Merge branch 'feat/37-1688-video-check' into integrate/37
This commit is contained in:
@@ -15,6 +15,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"cmsp/internal/alibaba1688"
|
||||
"cmsp/internal/config"
|
||||
"cmsp/internal/downloader"
|
||||
"cmsp/internal/erpgo"
|
||||
@@ -145,6 +146,13 @@ func (a *App) beforeClose(ctx context.Context) bool {
|
||||
} else if closed {
|
||||
a.log.Info("退出前已关闭淘宝专属 Chrome")
|
||||
}
|
||||
stateFile = config.ResolveDataPath("1688浏览器运行状态.json")
|
||||
closed, err = taobao.CloseBrowserOnExit(stateFile, a.cfg.Alibaba1688.UserDataDir)
|
||||
if err != nil {
|
||||
a.log.Warn("退出时保留未能确认归属或关闭的 1688 专属 Chrome:%v", err)
|
||||
} else if closed {
|
||||
a.log.Info("退出前已关闭 1688 专属 Chrome")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -432,6 +440,20 @@ func (a *App) FetchVideosForProduct(productID string) (FetchResult, error) {
|
||||
return FetchResult{ProductID: productID}, err
|
||||
}
|
||||
ctx := a.appContext()
|
||||
if a.cfg.Download.Source == "1688" {
|
||||
if err := alibaba1688.CheckFFmpeg(); err != nil {
|
||||
return FetchResult{ProductID: productID}, err
|
||||
}
|
||||
result, work, err := a.prepare1688Video(ctx, productID, a.cfg)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if work.Skipped {
|
||||
return result, nil
|
||||
}
|
||||
result.DownloadedCount, err = task.RunDownloads(ctx, 1, work)
|
||||
return result, err
|
||||
}
|
||||
riskGuard := task.NewEmptyRiskGuard(a.cfg.Download.RiskEmptyThreshold)
|
||||
result, work, err := a.prepareVideoFetch(ctx, productID, riskGuard, make(taobao.DetailCache), &taobao.LoginGuard{})
|
||||
if err != nil {
|
||||
@@ -917,6 +939,12 @@ func (a *App) StartVideoTask(productIDs []string) error {
|
||||
if err := downloader.CheckFFProbe(); err != nil {
|
||||
return err
|
||||
}
|
||||
batchConfig := a.cfg
|
||||
if batchConfig.Download.Source == "1688" {
|
||||
if err := alibaba1688.CheckFFmpeg(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
a.videoTaskMu.Lock()
|
||||
defer a.videoTaskMu.Unlock()
|
||||
@@ -948,6 +976,10 @@ func (a *App) StartVideoTask(productIDs []string) error {
|
||||
}, nil
|
||||
},
|
||||
Prepare: func(ctx context.Context, product task.Product) (task.Work, error) {
|
||||
if batchConfig.Download.Source == "1688" {
|
||||
_, work, err := a.prepare1688Video(ctx, product.ID, batchConfig)
|
||||
return work, err
|
||||
}
|
||||
_, work, err := a.prepareVideoFetch(ctx, product.ID, riskGuard, detailCache, loginGuard)
|
||||
return work, err
|
||||
},
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"cmsp/internal/alibaba1688"
|
||||
"cmsp/internal/config"
|
||||
"cmsp/internal/downloader"
|
||||
"cmsp/internal/store"
|
||||
"cmsp/internal/taobao"
|
||||
"cmsp/internal/task"
|
||||
)
|
||||
|
||||
// Open1688Login opens an isolated profile. The operator performs login manually.
|
||||
func (a *App) Open1688Login() error {
|
||||
state, err := taobao.EnsureBrowserWithState(a.appContext(), a.cfg.Alibaba1688, "https://www.1688.com/", "1688浏览器运行状态.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
page, err := taobao.Connect(state.Port)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer page.Close()
|
||||
if err := page.Navigate("https://www.1688.com/", 0); err != nil {
|
||||
return err
|
||||
}
|
||||
a.log.Info("已打开 1688 专属 Chrome,请在浏览器中手动完成登录")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) prepare1688Video(ctx context.Context, productID string, cfg config.Config) (FetchResult, task.Work, error) {
|
||||
result := FetchResult{ProductID: productID}
|
||||
product, found, err := a.db.GetProduct(productID)
|
||||
if err != nil {
|
||||
return result, task.Work{}, err
|
||||
}
|
||||
if !found {
|
||||
return result, task.Work{}, fmt.Errorf("找不到商品:%s", productID)
|
||||
}
|
||||
if product.DownloadStatus == store.DownloadDone {
|
||||
return result, task.Work{Skipped: true}, nil
|
||||
}
|
||||
if product.MainImage == "" {
|
||||
return result, task.Work{}, fmt.Errorf("商品没有可用于图搜的主图")
|
||||
}
|
||||
state, err := taobao.EnsureBrowserWithState(a.appContext(), cfg.Alibaba1688, "https://www.1688.com/", "1688浏览器运行状态.json")
|
||||
if err != nil {
|
||||
return result, task.Work{}, err
|
||||
}
|
||||
page, err := taobao.Connect(state.Port)
|
||||
if err != nil {
|
||||
return result, task.Work{}, err
|
||||
}
|
||||
offers, err := alibaba1688.SearchByImage(ctx, page, state.Port, product.MainImage, cfg.Download.SearchTopN)
|
||||
if err != nil {
|
||||
page.Close()
|
||||
if errors.Is(err, alibaba1688.ErrLoginRequired) {
|
||||
return result, task.Work{}, fmt.Errorf("%w:%v", task.ErrLoginRequired, err)
|
||||
}
|
||||
return result, task.Work{}, err
|
||||
}
|
||||
if len(offers) == 0 {
|
||||
page.Close()
|
||||
return result, task.Work{Skipped: true}, nil
|
||||
}
|
||||
root, err := filepath.Abs(cfg.Download.VideoDir)
|
||||
if err != nil {
|
||||
page.Close()
|
||||
return result, task.Work{}, err
|
||||
}
|
||||
productDir := filepath.Join(root, downloader.SafeDirName(product.ItemID))
|
||||
if err := os.MkdirAll(productDir, 0755); err != nil {
|
||||
page.Close()
|
||||
return result, task.Work{}, err
|
||||
}
|
||||
dl := downloader.New()
|
||||
var record store.Video
|
||||
var target string
|
||||
work := task.Work{Downloads: []task.DownloadFunc{func(downloadCtx context.Context) error {
|
||||
defer page.Close()
|
||||
for i, id := range offers {
|
||||
if err := downloadCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
videoURL, err := alibaba1688.OfferVideo(page, id, time.Duration(cfg.Download.DetailWaitSeconds*float64(time.Second)))
|
||||
if err != nil {
|
||||
if errors.Is(err, alibaba1688.ErrLoginRequired) {
|
||||
return fmt.Errorf("%w:%v", task.ErrLoginRequired, err)
|
||||
}
|
||||
a.log.Warn("蝦皮商品 %s 的 1688 候选 %s 打开失败:%v", product.ItemID, id, err)
|
||||
continue
|
||||
}
|
||||
if videoURL == "" {
|
||||
a.log.Info("蝦皮商品 %s 的 1688 候选 %s 没有可用视频", product.ItemID, id)
|
||||
continue
|
||||
}
|
||||
f, err := os.CreateTemp(productDir, ".cmsp-1688-*.mp4")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temp := f.Name()
|
||||
f.Close()
|
||||
os.Remove(temp)
|
||||
res, err := dl.Download(downloadCtx, videoURL, "https://detail.1688.com/offer/"+id+".html", temp, cfg.Download.DownloadRetries)
|
||||
if err != nil {
|
||||
os.Remove(temp)
|
||||
a.log.Warn("蝦皮商品 %s 的 1688 候选 %s 视频下载失败:%v", product.ItemID, id, err)
|
||||
continue
|
||||
}
|
||||
quality, reason, err := alibaba1688.Inspect(downloadCtx, temp)
|
||||
if err != nil || quality != alibaba1688.QualityUsable {
|
||||
os.Remove(temp)
|
||||
a.log.Warn("蝦皮商品 %s 的 1688 候选 %s 已跳过(%s,%s)", product.ItemID, id, quality, reason)
|
||||
continue
|
||||
}
|
||||
target = filepath.Join(productDir, downloader.Filename(product.ItemID, 1))
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
os.Remove(temp)
|
||||
return fmt.Errorf("目标视频文件已存在,保留原文件,未覆盖")
|
||||
}
|
||||
if err := os.Rename(temp, target); err != nil {
|
||||
os.Remove(temp)
|
||||
return err
|
||||
}
|
||||
record = store.Video{SourceItem: id, SourceURL: videoURL, LocalPath: target, FileSize: res.Size, Status: store.VideoStatusDownloaded, CreatedAt: time.Now().Format("2006-01-02 15:04:05")}
|
||||
a.log.Success("蝦皮商品 %s 的 1688 候选 %d/%d 通过逐条检测并下载", product.ItemID, i+1, len(offers))
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("1688 候选均无可用商品视频,原任务状态已保留")
|
||||
}}}
|
||||
work.Finalize = func(errs []error) error {
|
||||
if len(errs) == 0 || errs[0] != nil {
|
||||
return nil
|
||||
}
|
||||
if err := a.db.CompleteVideoDownload(product.ID, record); err != nil {
|
||||
os.Remove(target)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return result, work, nil
|
||||
}
|
||||
@@ -55,7 +55,16 @@ taobao:
|
||||
debug_port_start: 19666
|
||||
debug_port_end: 19765
|
||||
|
||||
alibaba1688:
|
||||
# 1688 单独的 Chrome Profile;在专属浏览器中手动登录。
|
||||
chrome_path: "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
|
||||
user_data_dir: "C:\\Users\\你的用户名\\AppData\\Local\\cmsp\\1688浏览器"
|
||||
debug_port_start: 19766
|
||||
debug_port_end: 19865
|
||||
|
||||
download:
|
||||
# taobao 或 1688;切换仅影响新启动的下载批次。
|
||||
source: taobao
|
||||
# 下载的视频保存到哪个目录。
|
||||
video_dir: "D:\\chengma\\cmsp\\运行数据\\视频"
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
|
||||
wiki_page: Architecture-and-Code-Map
|
||||
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Architecture-and-Code-Map.-
|
||||
wiki_revision: 9ebe51001ec9054656478312baedb3387d172c7a
|
||||
synchronized_at: 2026-10-05T10:01:39Z
|
||||
wiki_revision: 0ba3b5f3e6cf4630ef6fde05b0b17ba27a00b50b
|
||||
synchronized_at: 2026-10-07T07:49:50Z
|
||||
<!-- gitea-wiki-mirror:end -->
|
||||
|
||||
# 架构与代码地图
|
||||
@@ -180,3 +180,7 @@ cmsp/
|
||||
- **凭据不进入仓库。** 账号、密码、token、Cookie、Chrome Profile 内容不得出现在代码、日志、工单、Wiki 和提交中。
|
||||
- **SQLite 是任务状态的唯一事实来源。** 不允许再用 JSON 文件维护第二份任务状态。
|
||||
- **前端不做安全判断。** 按钮禁用只是提示,真正的校验必须在 Go 侧执行。
|
||||
|
||||
## 1688 视频下载来源(2026-10-07,#37)
|
||||
|
||||
商品列表“下载视频”旁选择淘宝或 1688;`download.source` 写入本机 `config.yaml`,批次启动时固定来源。1688 使用独立 Chrome Profile,操作者手动登录。`app_1688.go` 调用 `internal/alibaba1688/search.go`:上传商品主图到网页以图搜,按结果顺序进入候选详情,读取页面视频地址。`internal/downloader` 将候选 MP4 下载到临时文件;`internal/alibaba1688/quality.go` 用 ffprobe 校验、ffmpeg 抽帧检测黑底文字广告和静止拼图。只有判为 `usable` 的视频才晋升为商品文件,`internal/store.CompleteVideoDownload` 在同一 SQLite 事务中写视频记录与商品已下载状态。拒绝、无法判定、候选下载失败时保留原商品状态;上传仍沿用原有 erpgo 流程。
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
|
||||
wiki_page: Business-Rules-and-Glossary
|
||||
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Business-Rules-and-Glossary.-
|
||||
wiki_revision: 6deec49c2ba48aaf96a5842b4d525352afd6cf87
|
||||
synchronized_at: 2026-10-06T07:37:40Z
|
||||
wiki_revision: e05847d648d87a01f714ca7cb0af177e7630dcdf
|
||||
synchronized_at: 2026-10-07T07:49:50Z
|
||||
<!-- gitea-wiki-mirror:end -->
|
||||
|
||||
# 业务规则与术语
|
||||
@@ -149,3 +149,7 @@ MTOP 搜索方式、签名与请求参数保持现状。商品原链接以实际
|
||||
- Shopee 侧的实际生效延迟与验证方式。
|
||||
|
||||
每确认一项,就更新本页对应规则并删除此处条目;确认过程记录在对应工单中。
|
||||
|
||||
## 1688 作为可选视频来源(2026-10-07,#37)
|
||||
|
||||
下载来源由使用者在商品列表下载按钮旁选择并保存在本机配置,仅影响新批次。1688 与淘宝使用不同 Chrome Profile,登录由使用者手动完成。1688 图搜按结果顺序查找第一个有视频且通过内容检测的候选;每个候选先下载到临时文件,再检查时长、画面连续变化、黑底文字广告与静止图片。检测失败或无法可靠判定时删除临时文件,不写“已下载”;全部候选无可用视频时保留原状态,允许重试或换来源。通过检测后写入原有“已下载/待上传”状态,上传确认与 erpgo 上传流程不变。自动检测不能证明候选与目标商品完全同款,也不能证明视频使用权;批量上传的现有确认行为保持不变。
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
|
||||
wiki_page: Local-Development-and-Verification
|
||||
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Local-Development-and-Verification.-
|
||||
wiki_revision: 697d2e353c3b4b3f6648b7a08e4752235c42c7e7
|
||||
synchronized_at: 2026-10-06T07:37:41Z
|
||||
wiki_revision: 8c23a3a5b52f132282e9b6b19ae08b6fbb879c55
|
||||
synchronized_at: 2026-10-07T07:49:51Z
|
||||
<!-- gitea-wiki-mirror:end -->
|
||||
|
||||
# 本地开发与验证
|
||||
@@ -232,3 +232,7 @@ Go 实现完成后适用:
|
||||
6. 有工单的任务在工单追加最终证据评论,状态置为「待验收」,等待人工验收。
|
||||
|
||||
涉及货憨憨写操作、账号凭据、SQLite 结构变更、并发或删除数据时,必须先确认再执行,不能靠试错获得风险反馈。
|
||||
|
||||
## 1688 视频来源运行要求(2026-10-07,#37)
|
||||
|
||||
在商品列表下载按钮旁选“1688”,参数设置中检查 `alibaba1688.chrome_path`、独立 `user_data_dir` 与调试端口范围,点击“打开 1688 登录”并手动登录。1688 视频内容检测除了既有 ffprobe,还需要 `ffmpeg.exe`;打包运行时把二者放在 `cmsp.exe` 同目录,或加入 PATH。缺少 ffmpeg 时启动 1688 下载前即提示,不改变淘宝下载。`download.source` 保存在本机 `config.yaml`,默认 `taobao`。验证命令:`go test ./...`,`wails build`;真机验收分别用广告文字候选和正常商品视频检查拒绝、放行及 SQLite 状态。
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
|
||||
wiki_page: Troubleshooting
|
||||
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Troubleshooting
|
||||
wiki_revision: 0a176fa4cf7f3bd6a2c6b11ae3c6a58453500c85
|
||||
synchronized_at: 2026-10-05T02:28:58Z
|
||||
wiki_revision: 7b2d3a3a39b3fa1b8b90f564e6ad5c727771dad7
|
||||
synchronized_at: 2026-10-07T07:49:52Z
|
||||
<!-- gitea-wiki-mirror:end -->
|
||||
|
||||
# 故障排查
|
||||
@@ -142,3 +142,7 @@ SYNC_IN_PROGRESS 表示已有商品同步,请等待结束;Go侧拒绝重叠
|
||||
- 无法判断改动风险等级。
|
||||
|
||||
停止不等于失败。把现象、已确认事实和不确定的部分写清楚,比继续试更有价值。
|
||||
|
||||
## 1688 图搜与视频内容检测(2026-10-07,#37)
|
||||
|
||||
“1688 以图搜上传框未找到”或“图搜未打开结果页”时,先在独立 Chrome 中确认手动登录、首页正常显示和网页以图搜可用;不要自动绕过登录或安全验证。页面刚加载时控件和图片处理可能尚未就绪,程序会等待后再点击;持续失败时保留原任务状态,人工检查站点页面是否改版或访问受限。“1688 内容检测需要 ffmpeg.exe”时把 ffmpeg.exe 放在 cmsp.exe 同目录或加入 PATH,并保留既有 ffprobe。“候选均无可用商品视频”表示所查候选没有通过检测的 MP4,不表示该蝦皮商品在所有来源均无视频;可手动切换来源后重试。
|
||||
|
||||
@@ -62,6 +62,28 @@ const emptyQuery = () => ({
|
||||
pageSize: 20,
|
||||
})
|
||||
const query = ref(emptyQuery())
|
||||
const videoSource = ref('taobao')
|
||||
const sourceSaving = ref(false)
|
||||
const videoSourceOptions = [
|
||||
{ label: '淘宝', value: 'taobao' },
|
||||
{ label: '1688', value: '1688' },
|
||||
]
|
||||
|
||||
async function changeVideoSource(value) {
|
||||
if (taskRunning.value || sourceSaving.value) return
|
||||
sourceSaving.value = true
|
||||
try {
|
||||
const cfg = await AppBindings.GetConfig()
|
||||
cfg.download.source = value
|
||||
await AppBindings.SaveConfig(cfg)
|
||||
videoSource.value = value
|
||||
message.info(`新下载批次将使用${value === '1688' ? '1688' : '淘宝'}`)
|
||||
} catch (err) {
|
||||
message.error(`保存下载来源失败:${err}`)
|
||||
} finally {
|
||||
sourceSaving.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// 商品状态取值来自货憨憨接口文档,不要随便加。
|
||||
const statusOptions = [
|
||||
@@ -770,7 +792,7 @@ async function startVideoTask() {
|
||||
await StartVideoTask(selectedIDs)
|
||||
await refreshTaskProgress()
|
||||
startTaskPolling()
|
||||
message.info('批量取视频已启动')
|
||||
message.info(`批量取视频已启动,来源:${videoSource.value === '1688' ? '1688' : '淘宝'}`)
|
||||
} catch (err) {
|
||||
message.error(`启动批量取视频失败:${err}`)
|
||||
}
|
||||
@@ -882,6 +904,7 @@ async function onTaskLoginRequired(progress) {
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
AppBindings.GetConfig().then(cfg => { videoSource.value = cfg.download?.source || 'taobao' })
|
||||
// 先监听再读缓存:后台刷新若恰好此时完成,不会漏掉最新列表。
|
||||
EventsOn('shops:refreshed', onShopsRefreshed)
|
||||
EventsOn('shops:refresh-failed', onShopsRefreshFailed)
|
||||
@@ -976,6 +999,14 @@ onUnmounted(() => {
|
||||
>
|
||||
下载数据
|
||||
</n-button>
|
||||
<span class="lb">视频来源</span>
|
||||
<n-select
|
||||
:value="videoSource"
|
||||
:options="videoSourceOptions"
|
||||
:disabled="taskRunning || running !== '' || sourceSaving"
|
||||
style="width: 115px"
|
||||
@update:value="changeVideoSource"
|
||||
/>
|
||||
<n-button
|
||||
:loading="taskRunning"
|
||||
:disabled="taskRunning || running !== ''"
|
||||
|
||||
@@ -28,6 +28,7 @@ const dialog = useDialog()
|
||||
const cfg = ref(null)
|
||||
const saving = ref(false)
|
||||
const openingTaobao = ref(false)
|
||||
const opening1688 = ref(false)
|
||||
const checkingTaobao = ref(false)
|
||||
const taobaoStatus = ref(null)
|
||||
const resettingNoneProducts = ref(false)
|
||||
@@ -111,6 +112,19 @@ async function pickDir(field, title) {
|
||||
if (!picked) return
|
||||
if (field === 'userDataDir') cfg.value.taobao.userDataDir = picked
|
||||
if (field === 'videoDir') cfg.value.download.videoDir = picked
|
||||
if (field === 'alibaba1688UserDataDir') cfg.value.alibaba1688.userDataDir = picked
|
||||
}
|
||||
|
||||
async function open1688Login() {
|
||||
opening1688.value = true
|
||||
try {
|
||||
await AppBindings.Open1688Login()
|
||||
message.info('已打开 1688 专属 Chrome,请手动完成登录')
|
||||
} catch (err) {
|
||||
message.error(`打开 1688 失败:${err}`)
|
||||
} finally {
|
||||
opening1688.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function openTaobaoLogin() {
|
||||
@@ -272,6 +286,26 @@ onUnmounted(() => {
|
||||
</n-alert>
|
||||
</n-card>
|
||||
|
||||
<n-card title="1688 专属浏览器" size="small">
|
||||
<n-form label-placement="top" size="small">
|
||||
<n-form-item label="Chrome 可执行文件">
|
||||
<n-input v-model:value="cfg.alibaba1688.chromePath" />
|
||||
</n-form-item>
|
||||
<n-form-item label="独立用户数据目录(登录由你手动完成)">
|
||||
<n-input-group>
|
||||
<n-input v-model:value="cfg.alibaba1688.userDataDir" />
|
||||
<n-button @click="pickDir('alibaba1688UserDataDir', '选择 1688 Chrome 用户数据目录')">选择目录</n-button>
|
||||
</n-input-group>
|
||||
</n-form-item>
|
||||
<n-form-item label="调试端口范围">
|
||||
<n-input-number v-model:value="cfg.alibaba1688.debugPortStart" :show-button="false" style="width: 96px" />
|
||||
<span class="dash">—</span>
|
||||
<n-input-number v-model:value="cfg.alibaba1688.debugPortEnd" :show-button="false" style="width: 96px" />
|
||||
</n-form-item>
|
||||
</n-form>
|
||||
<n-button :loading="opening1688" @click="open1688Login">打开 1688 登录</n-button>
|
||||
</n-card>
|
||||
|
||||
<!-- 下载与任务 -->
|
||||
<n-card title="下载与任务" size="small">
|
||||
<n-form label-placement="top" size="small">
|
||||
|
||||
@@ -5,6 +5,7 @@ go 1.25.0
|
||||
require (
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/wailsapp/wails/v2 v2.15.0
|
||||
golang.org/x/sys v0.47.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.57.0
|
||||
)
|
||||
@@ -38,7 +39,6 @@ require (
|
||||
github.com/wailsapp/mimetype v1.4.1 // indirect
|
||||
golang.org/x/crypto v0.53.0 // indirect
|
||||
golang.org/x/net v0.56.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.39.0 // indirect
|
||||
modernc.org/libc v1.74.4 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package alibaba1688
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/png"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"cmsp/internal/downloader"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
type Quality string
|
||||
|
||||
const (
|
||||
QualityUsable Quality = "usable"
|
||||
QualityRejected Quality = "rejected"
|
||||
QualityUnknown Quality = "unknown"
|
||||
)
|
||||
|
||||
func CheckFFmpeg() error {
|
||||
_, err := findFFmpeg()
|
||||
if err != nil {
|
||||
return fmt.Errorf("1688 内容检测需要 ffmpeg.exe:请放在 cmsp.exe 同目录或加入 PATH")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func findFFmpeg() (string, error) {
|
||||
if executable, err := os.Executable(); err == nil {
|
||||
path := filepath.Join(filepath.Dir(executable), "ffmpeg.exe")
|
||||
if info, err := os.Stat(path); err == nil && !info.IsDir() {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
return exec.LookPath("ffmpeg")
|
||||
}
|
||||
|
||||
// Inspect decodes paired frames from the downloaded file. Only clear text cards
|
||||
// and near-static slide shows are rejected; ambiguous footage is never promoted.
|
||||
func Inspect(ctx context.Context, path string) (Quality, string, error) {
|
||||
probe, err := downloader.Probe(ctx, path)
|
||||
if err != nil {
|
||||
return QualityUnknown, "完整性校验失败", err
|
||||
}
|
||||
ffmpeg, err := findFFmpeg()
|
||||
if err != nil {
|
||||
return QualityUnknown, "缺少 ffmpeg", err
|
||||
}
|
||||
if probe.Duration < 3 {
|
||||
return QualityRejected, "视频时长过短", nil
|
||||
}
|
||||
var cards, moving, still int
|
||||
for _, part := range []float64{.1, .3, .5, .7, .9} {
|
||||
at := math.Min(probe.Duration-1, probe.Duration*part)
|
||||
a, err := sampleFrame(ctx, ffmpeg, path, at)
|
||||
if err != nil {
|
||||
return QualityUnknown, "抽帧失败", err
|
||||
}
|
||||
b, err := sampleFrame(ctx, ffmpeg, path, math.Min(probe.Duration-.1, at+.5))
|
||||
if err != nil {
|
||||
return QualityUnknown, "抽帧失败", err
|
||||
}
|
||||
if textCard(a) {
|
||||
cards++
|
||||
}
|
||||
if frameDifference(a, b) > 6 {
|
||||
moving++
|
||||
} else {
|
||||
still++
|
||||
}
|
||||
}
|
||||
if cards >= 4 {
|
||||
return QualityRejected, "多数画面是黑底文字广告", nil
|
||||
}
|
||||
if still >= 4 {
|
||||
return QualityRejected, "多数片段为静止图片", nil
|
||||
}
|
||||
if moving >= 2 && cards <= 1 {
|
||||
return QualityUsable, "检测到连续商品画面变化", nil
|
||||
}
|
||||
return QualityUnknown, "画面内容无法可靠判定", nil
|
||||
}
|
||||
|
||||
func sampleFrame(ctx context.Context, ffmpeg, path string, at float64) (image.Image, error) {
|
||||
cmd := exec.CommandContext(ctx, ffmpeg, "-hide_banner", "-loglevel", "error", "-ss", strconv.FormatFloat(at, 'f', 2, 64), "-i", path, "-frames:v", "1", "-vf", "scale=96:96", "-f", "image2pipe", "-vcodec", "png", "pipe:1")
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: windows.CREATE_NO_WINDOW}
|
||||
raw, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img, _, err := image.Decode(bytes.NewReader(raw))
|
||||
return img, err
|
||||
}
|
||||
|
||||
func textCard(img image.Image) bool {
|
||||
b := img.Bounds()
|
||||
total := b.Dx() * b.Dy()
|
||||
if total == 0 {
|
||||
return false
|
||||
}
|
||||
var dark, white, color int
|
||||
for y := b.Min.Y; y < b.Max.Y; y++ {
|
||||
for x := b.Min.X; x < b.Max.X; x++ {
|
||||
r, g, bl, _ := img.At(x, y).RGBA()
|
||||
rr, gg, bb := int(r>>8), int(g>>8), int(bl>>8)
|
||||
if rr < 45 && gg < 45 && bb < 45 {
|
||||
dark++
|
||||
}
|
||||
if rr > 180 && gg > 180 && bb > 180 {
|
||||
white++
|
||||
}
|
||||
if max3(rr, gg, bb)-min3(rr, gg, bb) > 35 {
|
||||
color++
|
||||
}
|
||||
}
|
||||
}
|
||||
return float64(dark)/float64(total) > .75 && float64(white)/float64(total) > .008 && float64(color)/float64(total) < .04
|
||||
}
|
||||
|
||||
func frameDifference(a, b image.Image) float64 {
|
||||
ra, rb := a.Bounds(), b.Bounds()
|
||||
if ra.Dx() != rb.Dx() || ra.Dy() != rb.Dy() {
|
||||
return 0
|
||||
}
|
||||
var sum float64
|
||||
for y := 0; y < ra.Dy(); y++ {
|
||||
for x := 0; x < ra.Dx(); x++ {
|
||||
r1, g1, b1, _ := a.At(ra.Min.X+x, ra.Min.Y+y).RGBA()
|
||||
r2, g2, b2, _ := b.At(rb.Min.X+x, rb.Min.Y+y).RGBA()
|
||||
sum += math.Abs(float64(int(r1>>8)-int(r2>>8))) + math.Abs(float64(int(g1>>8)-int(g2>>8))) + math.Abs(float64(int(b1>>8)-int(b2>>8)))
|
||||
}
|
||||
}
|
||||
return sum / float64(ra.Dx()*ra.Dy()*3)
|
||||
}
|
||||
|
||||
func max3(a, b, c int) int {
|
||||
if a < b {
|
||||
a = b
|
||||
}
|
||||
if a < c {
|
||||
a = c
|
||||
}
|
||||
return a
|
||||
}
|
||||
func min3(a, b, c int) int {
|
||||
if a > b {
|
||||
a = b
|
||||
}
|
||||
if a > c {
|
||||
a = c
|
||||
}
|
||||
return a
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package alibaba1688
|
||||
|
||||
import (
|
||||
"image"
|
||||
"image/color"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTextCardRejectsBlackBackgroundWithWhiteCopy(t *testing.T) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, 100, 100))
|
||||
for y := 0; y < 100; y++ {
|
||||
for x := 0; x < 100; x++ {
|
||||
if x >= 20 && x < 80 && y >= 40 && y < 50 {
|
||||
img.Set(x, y, color.RGBA{255, 255, 255, 255})
|
||||
} else {
|
||||
img.Set(x, y, color.RGBA{0, 0, 0, 255})
|
||||
}
|
||||
}
|
||||
}
|
||||
if !textCard(img) {
|
||||
t.Fatal("black-background sales card should be detected")
|
||||
}
|
||||
for y := 0; y < 100; y++ {
|
||||
for x := 0; x < 100; x++ {
|
||||
img.Set(x, y, color.RGBA{60, 130, 100, 255})
|
||||
}
|
||||
}
|
||||
if textCard(img) {
|
||||
t.Fatal("colored product frame should not be a sales card")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameDifferenceDetectsContinuousMotion(t *testing.T) {
|
||||
a := image.NewRGBA(image.Rect(0, 0, 20, 20))
|
||||
b := image.NewRGBA(image.Rect(0, 0, 20, 20))
|
||||
for y := 0; y < 20; y++ {
|
||||
for x := 0; x < 20; x++ {
|
||||
a.Set(x, y, color.RGBA{20, 20, 20, 255})
|
||||
b.Set(x, y, color.RGBA{100, 100, 100, 255})
|
||||
}
|
||||
}
|
||||
if frameDifference(a, a) != 0 || frameDifference(a, b) < 6 {
|
||||
t.Fatal("motion signal incorrect")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
package alibaba1688
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"cmsp/internal/taobao"
|
||||
)
|
||||
|
||||
var ErrLoginRequired = errors.New("1688 需要手动登录或完成安全验证")
|
||||
var offerIDPattern = regexp.MustCompile(`^[0-9]{8,20}$`)
|
||||
|
||||
type target struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
func targets(port int) ([]target, error) {
|
||||
client := http.Client{Timeout: 3 * time.Second}
|
||||
res, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/json/list", port))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取 1688 Chrome 页面失败: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
var values []target
|
||||
if err := json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(&values); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func fetchImage(ctx context.Context, raw string) (string, error) {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
|
||||
return "", fmt.Errorf("商品主图地址无效")
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("商品主图地址无效")
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0")
|
||||
client := http.Client{Timeout: 30 * time.Second}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("下载商品主图失败")
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != 200 {
|
||||
return "", fmt.Errorf("下载商品主图失败:HTTP %d", res.StatusCode)
|
||||
}
|
||||
f, err := os.CreateTemp("", "cmsp-1688-image-*.jpg")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
n, err := io.Copy(f, io.LimitReader(res.Body, (10<<20)+1))
|
||||
if err != nil || n > 10<<20 {
|
||||
os.Remove(f.Name())
|
||||
return "", fmt.Errorf("商品主图下载失败或超过 10 MB")
|
||||
}
|
||||
return f.Name(), nil
|
||||
}
|
||||
|
||||
// SearchByImage uses the site's visible upload control, then reads public result cards.
|
||||
// It never reads browser cookies or calls private search endpoints.
|
||||
func SearchByImage(ctx context.Context, page *taobao.CDP, port int, imageURL string, limit int) ([]string, error) {
|
||||
image, err := fetchImage(ctx, imageURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(image)
|
||||
before, err := targets(port)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, t := range before {
|
||||
seen[t.ID] = true
|
||||
}
|
||||
if err := page.Navigate("https://www.1688.com/", 3*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkPage(page, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var input struct {
|
||||
NodeID int `json:"nodeId"`
|
||||
}
|
||||
for i := 0; i < 10 && input.NodeID == 0; i++ {
|
||||
raw, err := page.Send("DOM.getDocument", map[string]any{"depth": 1})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var doc struct {
|
||||
Root struct {
|
||||
NodeID int `json:"nodeId"`
|
||||
} `json:"root"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err = page.Send("DOM.querySelector", map[string]any{"nodeId": doc.Root.NodeID, "selector": `input[type="file"][accept*="jpg"]`})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &input); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if input.NodeID == 0 {
|
||||
if err := wait(ctx, time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if input.NodeID == 0 {
|
||||
return nil, fmt.Errorf("1688 以图搜上传框未找到")
|
||||
}
|
||||
if _, err := page.Send("DOM.setFileInputFiles", map[string]any{"nodeId": input.NodeID, "files": []string{image}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var button string
|
||||
for i := 0; i < 10; i++ {
|
||||
if err := wait(ctx, time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
button, err = page.Evaluate(`(()=>{const b=[...document.querySelectorAll('button')].find(x=>x.textContent.trim()==='搜索'&&x.closest('[class*=imageSearch]'));if(!b||b.disabled||!b.closest('[class*=imageSearch]').innerText.includes('已上传1张图'))return '';const r=b.getBoundingClientRect();return JSON.stringify({x:r.x+r.width/2,y:r.y+r.height/2})})()`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if button != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if button == "" {
|
||||
return nil, fmt.Errorf("1688 以图搜图片未完成上传")
|
||||
}
|
||||
// The preview becomes visible before the site has finished preparing its search image.
|
||||
if err := wait(ctx, 2*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pos struct{ X, Y float64 }
|
||||
if err := json.Unmarshal([]byte(button), &pos); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := page.Send("Page.bringToFront", nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, typ := range []string{"mouseMoved", "mousePressed", "mouseReleased"} {
|
||||
if _, err := page.Send("Input.dispatchMouseEvent", map[string]any{"type": typ, "x": pos.X, "y": pos.Y, "button": "left", "clickCount": 1}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if typ == "mousePressed" {
|
||||
if err := wait(ctx, 100*time.Millisecond); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
var resultURL, resultID string
|
||||
for i := 0; i < 15 && resultURL == ""; i++ {
|
||||
if err := wait(ctx, time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list, err := targets(port)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, t := range list {
|
||||
if !seen[t.ID] && strings.HasPrefix(t.URL, "https://air.1688.com/kapp/1688-search/pc-image-search/") {
|
||||
resultURL, resultID = t.URL, t.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if resultURL == "" {
|
||||
return nil, fmt.Errorf("1688 图搜未打开结果页")
|
||||
}
|
||||
if err := page.Navigate(resultURL, 3*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resultID != "" {
|
||||
client := http.Client{Timeout: 2 * time.Second}
|
||||
if response, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/json/close/%s", port, resultID)); err == nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
}
|
||||
if err := checkPage(page, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if limit < 10 {
|
||||
limit = 10
|
||||
}
|
||||
expr := `JSON.stringify([...new Set([...document.querySelectorAll('[data-extra]')].map(x=>{try{return String(JSON.parse(x.dataset.extra).offerId||'')}catch(e){return ''}}).filter(Boolean))].slice(0,` + strconv.Itoa(limit) + `))`
|
||||
var offers []string
|
||||
for i := 0; i < 8; i++ {
|
||||
text, err := page.Evaluate(expr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(text), &offers); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(offers) > 0 {
|
||||
break
|
||||
}
|
||||
if err := wait(ctx, time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
valid := offers[:0]
|
||||
for _, id := range offers {
|
||||
if offerIDPattern.MatchString(id) {
|
||||
valid = append(valid, id)
|
||||
}
|
||||
}
|
||||
return valid, nil
|
||||
}
|
||||
|
||||
func OfferVideo(page *taobao.CDP, id string, delay time.Duration) (string, error) {
|
||||
if !offerIDPattern.MatchString(id) {
|
||||
return "", fmt.Errorf("1688 商品 ID 无效")
|
||||
}
|
||||
if err := page.Navigate("https://detail.1688.com/offer/"+id+".html", delay); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := checkPage(page, true); err != nil {
|
||||
return "", err
|
||||
}
|
||||
text, err := page.Evaluate(`JSON.stringify([...document.querySelectorAll('video')].map(v=>v.currentSrc||v.src||'').filter(Boolean))`)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var values []string
|
||||
if err := json.Unmarshal([]byte(text), &values); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, value := range values {
|
||||
u, err := url.Parse(value)
|
||||
if err != nil || u.Scheme != "https" || u.User != nil {
|
||||
continue
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
if host == "cloud.video.taobao.com" || strings.HasSuffix(host, ".cloudvideocdn.com") {
|
||||
return value, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func checkPage(page *taobao.CDP, detail bool) error {
|
||||
raw, err := page.Evaluate(`JSON.stringify({url:location.href,title:document.title,text:(document.body?.innerText||'').slice(0,1200)})`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var p struct{ URL, Title, Text string }
|
||||
if err := json.Unmarshal([]byte(raw), &p); err != nil {
|
||||
return err
|
||||
}
|
||||
u, err := url.Parse(p.URL)
|
||||
if err != nil || !strings.HasSuffix(strings.ToLower(u.Hostname()), "1688.com") {
|
||||
return ErrLoginRequired
|
||||
}
|
||||
if detail && (u.Hostname() != "detail.1688.com" || !strings.HasPrefix(u.Path, "/offer/")) {
|
||||
return ErrLoginRequired
|
||||
}
|
||||
for _, word := range []string{"访问被拒绝", "安全验证", "请输入验证码", "请先登录"} {
|
||||
if strings.Contains(p.Title, word) || strings.Contains(p.Text, word) {
|
||||
return ErrLoginRequired
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func wait(ctx context.Context, d time.Duration) error {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -35,11 +35,12 @@ import (
|
||||
// yaml tag 就是配置文件里的键名。改名会导致老配置读不出来,
|
||||
// 所以除非有充分理由,不要改已有字段的 tag。
|
||||
type Config struct {
|
||||
ERPGo ERPGoConfig `yaml:"erpgo" json:"erpgo"`
|
||||
Huohanhan HuohanhanConfig `yaml:"huohanhan" json:"huohanhan"`
|
||||
Taobao TaobaoConfig `yaml:"taobao" json:"taobao"`
|
||||
Download DownloadConfig `yaml:"download" json:"download"`
|
||||
Upload UploadConfig `yaml:"upload" json:"upload"`
|
||||
ERPGo ERPGoConfig `yaml:"erpgo" json:"erpgo"`
|
||||
Huohanhan HuohanhanConfig `yaml:"huohanhan" json:"huohanhan"`
|
||||
Taobao TaobaoConfig `yaml:"taobao" json:"taobao"`
|
||||
Alibaba1688 TaobaoConfig `yaml:"alibaba1688" json:"alibaba1688"`
|
||||
Download DownloadConfig `yaml:"download" json:"download"`
|
||||
Upload UploadConfig `yaml:"upload" json:"upload"`
|
||||
}
|
||||
|
||||
// ERPGoConfig 用于查询店铺和商品;APIKey 只保存在本机,禁止进入日志。
|
||||
@@ -90,6 +91,7 @@ type TaobaoConfig struct {
|
||||
|
||||
// DownloadConfig 是下载与任务参数。
|
||||
type DownloadConfig struct {
|
||||
Source string `yaml:"source" json:"source"`
|
||||
// VideoDir 是下载的视频保存目录。
|
||||
VideoDir string `yaml:"video_dir" json:"videoDir"`
|
||||
// MaxVideosPerProduct 是每个商品最多下载几个视频。
|
||||
@@ -131,7 +133,14 @@ func Default() Config {
|
||||
DebugPortStart: 19666,
|
||||
DebugPortEnd: 19765,
|
||||
},
|
||||
Alibaba1688: TaobaoConfig{
|
||||
ChromePath: `C:\Program Files\Google\Chrome\Application\chrome.exe`,
|
||||
UserDataDir: filepath.Join(os.Getenv("LOCALAPPDATA"), "cmsp", "1688浏览器"),
|
||||
DebugPortStart: 19766,
|
||||
DebugPortEnd: 19865,
|
||||
},
|
||||
Download: DownloadConfig{
|
||||
Source: "taobao",
|
||||
VideoDir: DefaultVideoDir(),
|
||||
MaxVideosPerProduct: 3,
|
||||
SearchTopN: 5,
|
||||
@@ -195,6 +204,24 @@ func (c Config) Validate() error {
|
||||
}
|
||||
// 旧 huohanhan 配置只为兼容现有 config.yaml 保留,不再用于请求。
|
||||
t := c.Taobao
|
||||
if c.Download.Source != "taobao" && c.Download.Source != "1688" {
|
||||
return fmt.Errorf("下载来源须为 taobao 或 1688")
|
||||
}
|
||||
if strings.TrimSpace(c.Alibaba1688.ChromePath) == "" || strings.TrimSpace(c.Alibaba1688.UserDataDir) == "" {
|
||||
return fmt.Errorf("1688 专属 Chrome 路径和用户数据目录不能为空")
|
||||
}
|
||||
if filepath.Clean(c.Alibaba1688.UserDataDir) == filepath.Clean(t.UserDataDir) {
|
||||
return fmt.Errorf("1688 和淘宝不能共用 Chrome 用户数据目录")
|
||||
}
|
||||
if err := checkPort(c.Alibaba1688.DebugPortStart, "1688 调试端口起始"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkPort(c.Alibaba1688.DebugPortEnd, "1688 调试端口结束"); err != nil {
|
||||
return err
|
||||
}
|
||||
if c.Alibaba1688.DebugPortEnd < c.Alibaba1688.DebugPortStart {
|
||||
return fmt.Errorf("1688 调试端口结束不能小于起始端口")
|
||||
}
|
||||
if strings.TrimSpace(t.ChromePath) == "" {
|
||||
return fmt.Errorf("Chrome 可执行文件路径不能为空")
|
||||
}
|
||||
@@ -375,7 +402,16 @@ taobao:
|
||||
debug_port_start: %d
|
||||
debug_port_end: %d
|
||||
|
||||
alibaba1688:
|
||||
# 1688 使用独立 Profile,登录只能在专属 Chrome 中手动完成。
|
||||
chrome_path: %s
|
||||
user_data_dir: %s
|
||||
debug_port_start: %d
|
||||
debug_port_end: %d
|
||||
|
||||
download:
|
||||
# 下载来源:taobao 或 1688;仅影响新批次。
|
||||
source: %s
|
||||
# 下载的视频保存到哪个目录。
|
||||
video_dir: %s
|
||||
|
||||
@@ -421,6 +457,11 @@ upload:
|
||||
quoted(c.Taobao.UserDataDir),
|
||||
c.Taobao.DebugPortStart,
|
||||
c.Taobao.DebugPortEnd,
|
||||
quoted(c.Alibaba1688.ChromePath),
|
||||
quoted(c.Alibaba1688.UserDataDir),
|
||||
c.Alibaba1688.DebugPortStart,
|
||||
c.Alibaba1688.DebugPortEnd,
|
||||
quoted(c.Download.Source),
|
||||
quoted(c.Download.VideoDir),
|
||||
c.Download.MaxVideosPerProduct,
|
||||
c.Download.SearchTopN,
|
||||
|
||||
@@ -53,6 +53,26 @@ func TestDefaultConfigIsValid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoSourcePersistsAndKeepsIndependentBrowser(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
cfg := Default()
|
||||
cfg.Download.Source = "1688"
|
||||
if cfg.Alibaba1688.UserDataDir == cfg.Taobao.UserDataDir {
|
||||
t.Fatal("1688 and Taobao must not share Chrome profile")
|
||||
}
|
||||
if err := Save(path, cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := Load(path)
|
||||
if err != nil || loaded.Download.Source != "1688" || loaded.Alibaba1688.UserDataDir != cfg.Alibaba1688.UserDataDir {
|
||||
t.Fatalf("source/browser config not persisted: %v", err)
|
||||
}
|
||||
cfg.Download.Source = "unknown"
|
||||
if cfg.Validate() == nil {
|
||||
t.Fatal("unsupported source accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsBadValues(t *testing.T) {
|
||||
// 每个用例只改一个字段,这样失败时能一眼看出是哪条规则出问题。
|
||||
cases := []struct {
|
||||
|
||||
+34
-3
@@ -1,6 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -34,6 +35,39 @@ func (s *Store) ReplaceVideos(productID string, items []Video, now string) error
|
||||
return fmt.Errorf("开启视频替换事务失败:%w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := replaceVideosTx(tx, productID, items, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("提交视频替换事务失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CompleteVideoDownload commits the accepted video and product status together.
|
||||
func (s *Store) CompleteVideoDownload(productID string, video Video) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("开启视频下载完成事务失败:%w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := replaceVideosTx(tx, productID, []Video{video}, video.CreatedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := tx.Exec(`UPDATE products SET video_status = ?, download_status = ?, last_error = '' WHERE id = ?`, VideoFound, DownloadDone, productID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("更新商品下载状态失败:%w", err)
|
||||
}
|
||||
if count, err := result.RowsAffected(); err != nil || count != 1 {
|
||||
return fmt.Errorf("商品下载状态更新数量异常:%d,%v", count, err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("提交视频下载完成事务失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func replaceVideosTx(tx *sql.Tx, productID string, items []Video, now string) error {
|
||||
if _, err := tx.Exec(`DELETE FROM videos WHERE product_id = ?`, productID); err != nil {
|
||||
return fmt.Errorf("清理商品旧视频失败:%w", err)
|
||||
}
|
||||
@@ -56,9 +90,6 @@ func (s *Store) ReplaceVideos(productID string, items []Video, now string) error
|
||||
return fmt.Errorf("写入商品视频失败:%w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("提交视频替换事务失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,31 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCompleteVideoDownloadCommitsVideoAndStatusTogether(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
if _, err := s.DB().Exec(`INSERT INTO products (id, item_id, video_status, download_status) VALUES ('p1', 'fictional-1', 'pending', 'pending')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
video := Video{SourceItem: "fictional-1688-1", LocalPath: "D:/fictional.mp4", Status: VideoStatusDownloaded, CreatedAt: "2026-10-07 12:00:00"}
|
||||
if err := s.CompleteVideoDownload("missing", video); err == nil {
|
||||
t.Fatal("missing product must roll back")
|
||||
}
|
||||
if items, err := s.ListVideos("missing"); err != nil || len(items) != 0 {
|
||||
t.Fatalf("rollback failed: %v %+v", err, items)
|
||||
}
|
||||
if err := s.CompleteVideoDownload("p1", video); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
product, found, err := s.GetProduct("p1")
|
||||
if err != nil || !found || product.VideoStatus != VideoFound || product.DownloadStatus != DownloadDone {
|
||||
t.Fatalf("status mismatch: %+v %v %v", product, found, err)
|
||||
}
|
||||
items, err := s.ListVideos("p1")
|
||||
if err != nil || len(items) != 1 || items[0].SourceItem != video.SourceItem {
|
||||
t.Fatalf("video mismatch: %+v %v", items, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplaceVideos按商品全量替换(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
first := []Video{
|
||||
|
||||
@@ -35,6 +35,11 @@ type BrowserState struct {
|
||||
|
||||
// EnsureBrowser 复用归属明确且可连接的专属 Chrome;否则安全地启动一个新实例。
|
||||
func EnsureBrowser(ctx context.Context, cfg config.TaobaoConfig, openURL string) (BrowserState, error) {
|
||||
return EnsureBrowserWithState(ctx, cfg, openURL, browserStateName)
|
||||
}
|
||||
|
||||
// EnsureBrowserWithState isolates runtime ownership metadata for another site/profile.
|
||||
func EnsureBrowserWithState(ctx context.Context, cfg config.TaobaoConfig, openURL, stateName string) (BrowserState, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
@@ -51,7 +56,10 @@ func EnsureBrowser(ctx context.Context, cfg config.TaobaoConfig, openURL string)
|
||||
return BrowserState{}, fmt.Errorf("创建 Chrome 用户数据目录失败:%w", err)
|
||||
}
|
||||
|
||||
stateFile := config.ResolveDataPath(browserStateName)
|
||||
if stateName != browserStateName && stateName != "1688浏览器运行状态.json" {
|
||||
return BrowserState{}, fmt.Errorf("不支持的浏览器状态文件")
|
||||
}
|
||||
stateFile := config.ResolveDataPath(stateName)
|
||||
state, found, readErr := readBrowserState(stateFile)
|
||||
if readErr == nil && found && browserStateReusable(state, profile) {
|
||||
return state, nil
|
||||
|
||||
@@ -264,6 +264,14 @@ func (r *Runner) run(ctx context.Context, queue []queuedProduct) {
|
||||
r.completeProduct(0, err == nil, err != nil)
|
||||
} else {
|
||||
downloaded, downloadErr := RunDownloads(ctx, r.options.Concurrency, work)
|
||||
if errors.Is(downloadErr, ErrLoginRequired) || errors.Is(downloadErr, ErrRiskBlocked) {
|
||||
r.setStopReason("login_required")
|
||||
if errors.Is(downloadErr, ErrRiskBlocked) {
|
||||
r.setStopReason(StopReasonRiskBlocked)
|
||||
}
|
||||
r.finish(StateLoginRequired)
|
||||
return
|
||||
}
|
||||
r.completeProduct(downloaded, false, downloadErr != nil)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user