From 2d125a655e58cc093ad3355ac9f0b0150997f96c Mon Sep 17 00:00:00 2001 From: QiuSW Date: Wed, 7 Oct 2026 15:51:28 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=201688=20=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E6=9D=A5=E6=BA=90=E5=8F=8A=E9=80=90=E6=9D=A1=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E6=A3=80=E6=B5=8B=20(#37)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app.go | 32 ++ app_1688.go | 148 +++++++++ config.example.yaml | 9 + docs/02-architecture-and-code-map.md | 8 +- docs/03-business-rules-and-glossary.md | 8 +- docs/04-local-development-and-verification.md | 8 +- docs/06-troubleshooting.md | 8 +- frontend/src/views/ProductListView.vue | 33 +- frontend/src/views/SettingsView.vue | 34 ++ go.mod | 2 +- internal/alibaba1688/quality.go | 161 ++++++++++ internal/alibaba1688/quality_test.go | 45 +++ internal/alibaba1688/search.go | 292 ++++++++++++++++++ internal/config/config.go | 51 ++- internal/config/config_test.go | 20 ++ internal/store/video.go | 37 ++- internal/store/video_test.go | 25 ++ internal/taobao/chrome.go | 10 +- internal/task/task.go | 8 + 19 files changed, 920 insertions(+), 19 deletions(-) create mode 100644 app_1688.go create mode 100644 internal/alibaba1688/quality.go create mode 100644 internal/alibaba1688/quality_test.go create mode 100644 internal/alibaba1688/search.go diff --git a/app.go b/app.go index fe21050..464af1e 100644 --- a/app.go +++ b/app.go @@ -15,6 +15,7 @@ import ( "sync" "time" + "cmsp/internal/alibaba1688" "cmsp/internal/config" "cmsp/internal/downloader" "cmsp/internal/erpgo" @@ -145,6 +146,13 @@ func (a *App) beforeClose(ctx context.Context) bool { } else if closed { a.log.Info("退出前已关闭淘宝专属 Chrome") } + stateFile = config.ResolveDataPath("1688浏览器运行状态.json") + closed, err = taobao.CloseBrowserOnExit(stateFile, a.cfg.Alibaba1688.UserDataDir) + if err != nil { + a.log.Warn("退出时保留未能确认归属或关闭的 1688 专属 Chrome：%v", err) + } else if closed { + a.log.Info("退出前已关闭 1688 专属 Chrome") + } return false } @@ -432,6 +440,20 @@ func (a *App) FetchVideosForProduct(productID string) (FetchResult, error) { return FetchResult{ProductID: productID}, err } ctx := a.appContext() + if a.cfg.Download.Source == "1688" { + if err := alibaba1688.CheckFFmpeg(); err != nil { + return FetchResult{ProductID: productID}, err + } + result, work, err := a.prepare1688Video(ctx, productID, a.cfg) + if err != nil { + return result, err + } + if work.Skipped { + return result, nil + } + result.DownloadedCount, err = task.RunDownloads(ctx, 1, work) + return result, err + } riskGuard := task.NewEmptyRiskGuard(a.cfg.Download.RiskEmptyThreshold) result, work, err := a.prepareVideoFetch(ctx, productID, riskGuard, make(taobao.DetailCache), &taobao.LoginGuard{}) if err != nil { @@ -917,6 +939,12 @@ func (a *App) StartVideoTask(productIDs []string) error { if err := downloader.CheckFFProbe(); err != nil { return err } + batchConfig := a.cfg + if batchConfig.Download.Source == "1688" { + if err := alibaba1688.CheckFFmpeg(); err != nil { + return err + } + } a.videoTaskMu.Lock() defer a.videoTaskMu.Unlock() @@ -948,6 +976,10 @@ func (a *App) StartVideoTask(productIDs []string) error { }, nil }, Prepare: func(ctx context.Context, product task.Product) (task.Work, error) { + if batchConfig.Download.Source == "1688" { + _, work, err := a.prepare1688Video(ctx, product.ID, batchConfig) + return work, err + } _, work, err := a.prepareVideoFetch(ctx, product.ID, riskGuard, detailCache, loginGuard) return work, err }, diff --git a/app_1688.go b/app_1688.go new file mode 100644 index 0000000..fcbaf86 --- /dev/null +++ b/app_1688.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "cmsp/internal/alibaba1688" + "cmsp/internal/config" + "cmsp/internal/downloader" + "cmsp/internal/store" + "cmsp/internal/taobao" + "cmsp/internal/task" +) + +// Open1688Login opens an isolated profile. The operator performs login manually. +func (a *App) Open1688Login() error { + state, err := taobao.EnsureBrowserWithState(a.appContext(), a.cfg.Alibaba1688, "https://www.1688.com/", "1688浏览器运行状态.json") + if err != nil { + return err + } + page, err := taobao.Connect(state.Port) + if err != nil { + return err + } + defer page.Close() + if err := page.Navigate("https://www.1688.com/", 0); err != nil { + return err + } + a.log.Info("已打开 1688 专属 Chrome，请在浏览器中手动完成登录") + return nil +} + +func (a *App) prepare1688Video(ctx context.Context, productID string, cfg config.Config) (FetchResult, task.Work, error) { + result := FetchResult{ProductID: productID} + product, found, err := a.db.GetProduct(productID) + if err != nil { + return result, task.Work{}, err + } + if !found { + return result, task.Work{}, fmt.Errorf("找不到商品：%s", productID) + } + if product.DownloadStatus == store.DownloadDone { + return result, task.Work{Skipped: true}, nil + } + if product.MainImage == "" { + return result, task.Work{}, fmt.Errorf("商品没有可用于图搜的主图") + } + state, err := taobao.EnsureBrowserWithState(a.appContext(), cfg.Alibaba1688, "https://www.1688.com/", "1688浏览器运行状态.json") + if err != nil { + return result, task.Work{}, err + } + page, err := taobao.Connect(state.Port) + if err != nil { + return result, task.Work{}, err + } + offers, err := alibaba1688.SearchByImage(ctx, page, state.Port, product.MainImage, cfg.Download.SearchTopN) + if err != nil { + page.Close() + if errors.Is(err, alibaba1688.ErrLoginRequired) { + return result, task.Work{}, fmt.Errorf("%w：%v", task.ErrLoginRequired, err) + } + return result, task.Work{}, err + } + if len(offers) == 0 { + page.Close() + return result, task.Work{Skipped: true}, nil + } + root, err := filepath.Abs(cfg.Download.VideoDir) + if err != nil { + page.Close() + return result, task.Work{}, err + } + productDir := filepath.Join(root, downloader.SafeDirName(product.ItemID)) + if err := os.MkdirAll(productDir, 0755); err != nil { + page.Close() + return result, task.Work{}, err + } + dl := downloader.New() + var record store.Video + var target string + work := task.Work{Downloads: []task.DownloadFunc{func(downloadCtx context.Context) error { + defer page.Close() + for i, id := range offers { + if err := downloadCtx.Err(); err != nil { + return err + } + videoURL, err := alibaba1688.OfferVideo(page, id, time.Duration(cfg.Download.DetailWaitSeconds*float64(time.Second))) + if err != nil { + if errors.Is(err, alibaba1688.ErrLoginRequired) { + return fmt.Errorf("%w：%v", task.ErrLoginRequired, err) + } + a.log.Warn("蝦皮商品 %s 的 1688 候选 %s 打开失败：%v", product.ItemID, id, err) + continue + } + if videoURL == "" { + a.log.Info("蝦皮商品 %s 的 1688 候选 %s 没有可用视频", product.ItemID, id) + continue + } + f, err := os.CreateTemp(productDir, ".cmsp-1688-*.mp4") + if err != nil { + return err + } + temp := f.Name() + f.Close() + os.Remove(temp) + res, err := dl.Download(downloadCtx, videoURL, "https://detail.1688.com/offer/"+id+".html", temp, cfg.Download.DownloadRetries) + if err != nil { + os.Remove(temp) + a.log.Warn("蝦皮商品 %s 的 1688 候选 %s 视频下载失败：%v", product.ItemID, id, err) + continue + } + quality, reason, err := alibaba1688.Inspect(downloadCtx, temp) + if err != nil || quality != alibaba1688.QualityUsable { + os.Remove(temp) + a.log.Warn("蝦皮商品 %s 的 1688 候选 %s 已跳过（%s，%s）", product.ItemID, id, quality, reason) + continue + } + target = filepath.Join(productDir, downloader.Filename(product.ItemID, 1)) + if _, err := os.Stat(target); err == nil { + os.Remove(temp) + return fmt.Errorf("目标视频文件已存在，保留原文件，未覆盖") + } + if err := os.Rename(temp, target); err != nil { + os.Remove(temp) + return err + } + record = store.Video{SourceItem: id, SourceURL: videoURL, LocalPath: target, FileSize: res.Size, Status: store.VideoStatusDownloaded, CreatedAt: time.Now().Format("2006-01-02 15:04:05")} + a.log.Success("蝦皮商品 %s 的 1688 候选 %d/%d 通过逐条检测并下载", product.ItemID, i+1, len(offers)) + return nil + } + return fmt.Errorf("1688 候选均无可用商品视频，原任务状态已保留") + }}} + work.Finalize = func(errs []error) error { + if len(errs) == 0 || errs[0] != nil { + return nil + } + if err := a.db.CompleteVideoDownload(product.ID, record); err != nil { + os.Remove(target) + return err + } + return nil + } + return result, work, nil +} diff --git a/config.example.yaml b/config.example.yaml index 0ac139f..445bfca 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -55,7 +55,16 @@ taobao: debug_port_start: 19666 debug_port_end: 19765 +alibaba1688: + # 1688 单独的 Chrome Profile；在专属浏览器中手动登录。 + chrome_path: "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" + user_data_dir: "C:\\Users\\你的用户名\\AppData\\Local\\cmsp\\1688浏览器" + debug_port_start: 19766 + debug_port_end: 19865 + download: + # taobao 或 1688；切换仅影响新启动的下载批次。 + source: taobao # 下载的视频保存到哪个目录。 video_dir: "D:\\chengma\\cmsp\\运行数据\\视频" diff --git a/docs/02-architecture-and-code-map.md b/docs/02-architecture-and-code-map.md index 5917961..1f9c82f 100644 --- a/docs/02-architecture-and-code-map.md +++ b/docs/02-architecture-and-code-map.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Architecture-and-Code-Map wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Architecture-and-Code-Map.- -wiki_revision: 9ebe51001ec9054656478312baedb3387d172c7a -synchronized_at: 2026-10-05T10:01:39Z +wiki_revision: 0ba3b5f3e6cf4630ef6fde05b0b17ba27a00b50b +synchronized_at: 2026-10-07T07:49:50Z # 架构与代码地图 @@ -180,3 +180,7 @@ cmsp/ - **凭据不进入仓库。** 账号、密码、token、Cookie、Chrome Profile 内容不得出现在代码、日志、工单、Wiki 和提交中。 - **SQLite 是任务状态的唯一事实来源。** 不允许再用 JSON 文件维护第二份任务状态。 - **前端不做安全判断。** 按钮禁用只是提示，真正的校验必须在 Go 侧执行。 + +## 1688 视频下载来源（2026-10-07，#37） + +商品列表“下载视频”旁选择淘宝或 1688；`download.source` 写入本机 `config.yaml`，批次启动时固定来源。1688 使用独立 Chrome Profile，操作者手动登录。`app_1688.go` 调用 `internal/alibaba1688/search.go`：上传商品主图到网页以图搜，按结果顺序进入候选详情，读取页面视频地址。`internal/downloader` 将候选 MP4 下载到临时文件；`internal/alibaba1688/quality.go` 用 ffprobe 校验、ffmpeg 抽帧检测黑底文字广告和静止拼图。只有判为 `usable` 的视频才晋升为商品文件，`internal/store.CompleteVideoDownload` 在同一 SQLite 事务中写视频记录与商品已下载状态。拒绝、无法判定、候选下载失败时保留原商品状态；上传仍沿用原有 erpgo 流程。 diff --git a/docs/03-business-rules-and-glossary.md b/docs/03-business-rules-and-glossary.md index 5a7f318..06a5229 100644 --- a/docs/03-business-rules-and-glossary.md +++ b/docs/03-business-rules-and-glossary.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Business-Rules-and-Glossary wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Business-Rules-and-Glossary.- -wiki_revision: 6deec49c2ba48aaf96a5842b4d525352afd6cf87 -synchronized_at: 2026-10-06T07:37:40Z +wiki_revision: e05847d648d87a01f714ca7cb0af177e7630dcdf +synchronized_at: 2026-10-07T07:49:50Z # 业务规则与术语 @@ -149,3 +149,7 @@ MTOP 搜索方式、签名与请求参数保持现状。商品原链接以实际 - Shopee 侧的实际生效延迟与验证方式。 每确认一项，就更新本页对应规则并删除此处条目；确认过程记录在对应工单中。 + +## 1688 作为可选视频来源（2026-10-07，#37） + +下载来源由使用者在商品列表下载按钮旁选择并保存在本机配置，仅影响新批次。1688 与淘宝使用不同 Chrome Profile，登录由使用者手动完成。1688 图搜按结果顺序查找第一个有视频且通过内容检测的候选；每个候选先下载到临时文件，再检查时长、画面连续变化、黑底文字广告与静止图片。检测失败或无法可靠判定时删除临时文件，不写“已下载”；全部候选无可用视频时保留原状态，允许重试或换来源。通过检测后写入原有“已下载/待上传”状态，上传确认与 erpgo 上传流程不变。自动检测不能证明候选与目标商品完全同款，也不能证明视频使用权；批量上传的现有确认行为保持不变。 diff --git a/docs/04-local-development-and-verification.md b/docs/04-local-development-and-verification.md index aacace8..c0221c3 100644 --- a/docs/04-local-development-and-verification.md +++ b/docs/04-local-development-and-verification.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Local-Development-and-Verification wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Local-Development-and-Verification.- -wiki_revision: 697d2e353c3b4b3f6648b7a08e4752235c42c7e7 -synchronized_at: 2026-10-06T07:37:41Z +wiki_revision: 8c23a3a5b52f132282e9b6b19ae08b6fbb879c55 +synchronized_at: 2026-10-07T07:49:51Z # 本地开发与验证 @@ -232,3 +232,7 @@ Go 实现完成后适用： 6. 有工单的任务在工单追加最终证据评论，状态置为「待验收」，等待人工验收。 涉及货憨憨写操作、账号凭据、SQLite 结构变更、并发或删除数据时，必须先确认再执行，不能靠试错获得风险反馈。 + +## 1688 视频来源运行要求（2026-10-07，#37） + +在商品列表下载按钮旁选“1688”，参数设置中检查 `alibaba1688.chrome_path`、独立 `user_data_dir` 与调试端口范围，点击“打开 1688 登录”并手动登录。1688 视频内容检测除了既有 ffprobe，还需要 `ffmpeg.exe`；打包运行时把二者放在 `cmsp.exe` 同目录，或加入 PATH。缺少 ffmpeg 时启动 1688 下载前即提示，不改变淘宝下载。`download.source` 保存在本机 `config.yaml`，默认 `taobao`。验证命令：`go test ./...`，`wails build`；真机验收分别用广告文字候选和正常商品视频检查拒绝、放行及 SQLite 状态。 diff --git a/docs/06-troubleshooting.md b/docs/06-troubleshooting.md index bd141ab..20f0ae0 100644 --- a/docs/06-troubleshooting.md +++ b/docs/06-troubleshooting.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Troubleshooting wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Troubleshooting -wiki_revision: 0a176fa4cf7f3bd6a2c6b11ae3c6a58453500c85 -synchronized_at: 2026-10-05T02:28:58Z +wiki_revision: 7b2d3a3a39b3fa1b8b90f564e6ad5c727771dad7 +synchronized_at: 2026-10-07T07:49:52Z # 故障排查 @@ -142,3 +142,7 @@ SYNC_IN_PROGRESS 表示已有商品同步，请等待结束；Go侧拒绝重叠 - 无法判断改动风险等级。 停止不等于失败。把现象、已确认事实和不确定的部分写清楚，比继续试更有价值。 + +## 1688 图搜与视频内容检测（2026-10-07，#37） + +“1688 以图搜上传框未找到”或“图搜未打开结果页”时，先在独立 Chrome 中确认手动登录、首页正常显示和网页以图搜可用；不要自动绕过登录或安全验证。页面刚加载时控件和图片处理可能尚未就绪，程序会等待后再点击；持续失败时保留原任务状态，人工检查站点页面是否改版或访问受限。“1688 内容检测需要 ffmpeg.exe”时把 ffmpeg.exe 放在 cmsp.exe 同目录或加入 PATH，并保留既有 ffprobe。“候选均无可用商品视频”表示所查候选没有通过检测的 MP4，不表示该蝦皮商品在所有来源均无视频；可手动切换来源后重试。 diff --git a/frontend/src/views/ProductListView.vue b/frontend/src/views/ProductListView.vue index 398a034..1d8dcf6 100644 --- a/frontend/src/views/ProductListView.vue +++ b/frontend/src/views/ProductListView.vue @@ -62,6 +62,28 @@ const emptyQuery = () => ({ pageSize: 20, }) const query = ref(emptyQuery()) +const videoSource = ref('taobao') +const sourceSaving = ref(false) +const videoSourceOptions = [ + { label: '淘宝', value: 'taobao' }, + { label: '1688', value: '1688' }, +] + +async function changeVideoSource(value) { + if (taskRunning.value || sourceSaving.value) return + sourceSaving.value = true + try { + const cfg = await AppBindings.GetConfig() + cfg.download.source = value + await AppBindings.SaveConfig(cfg) + videoSource.value = value + message.info(`新下载批次将使用${value === '1688' ? '1688' : '淘宝'}`) + } catch (err) { + message.error(`保存下载来源失败：${err}`) + } finally { + sourceSaving.value = false + } +} // 商品状态取值来自货憨憨接口文档，不要随便加。 const statusOptions = [ @@ -770,7 +792,7 @@ async function startVideoTask() { await StartVideoTask(selectedIDs) await refreshTaskProgress() startTaskPolling() - message.info('批量取视频已启动') + message.info(`批量取视频已启动，来源：${videoSource.value === '1688' ? '1688' : '淘宝'}`) } catch (err) { message.error(`启动批量取视频失败：${err}`) } @@ -882,6 +904,7 @@ async function onTaskLoginRequired(progress) { } onMounted(() => { + AppBindings.GetConfig().then(cfg => { videoSource.value = cfg.download?.source || 'taobao' }) // 先监听再读缓存：后台刷新若恰好此时完成，不会漏掉最新列表。 EventsOn('shops:refreshed', onShopsRefreshed) EventsOn('shops:refresh-failed', onShopsRefreshFailed) @@ -976,6 +999,14 @@ onUnmounted(() => { > 下载数据 + 视频来源 + { + + + + + + + + + 选择目录 + + + + + — + + + + 打开 1688 登录 + + diff --git a/go.mod b/go.mod index ee1f0c7..c9c50e0 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.0 require ( github.com/gorilla/websocket v1.5.3 github.com/wailsapp/wails/v2 v2.15.0 + golang.org/x/sys v0.47.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.57.0 ) @@ -38,7 +39,6 @@ require ( github.com/wailsapp/mimetype v1.4.1 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/net v0.56.0 // indirect - golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.39.0 // indirect modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/internal/alibaba1688/quality.go b/internal/alibaba1688/quality.go new file mode 100644 index 0000000..c264eab --- /dev/null +++ b/internal/alibaba1688/quality.go @@ -0,0 +1,161 @@ +package alibaba1688 + +import ( + "bytes" + "context" + "fmt" + "image" + _ "image/png" + "math" + "os" + "os/exec" + "path/filepath" + "strconv" + "syscall" + + "cmsp/internal/downloader" + "golang.org/x/sys/windows" +) + +type Quality string + +const ( + QualityUsable Quality = "usable" + QualityRejected Quality = "rejected" + QualityUnknown Quality = "unknown" +) + +func CheckFFmpeg() error { + _, err := findFFmpeg() + if err != nil { + return fmt.Errorf("1688 内容检测需要 ffmpeg.exe：请放在 cmsp.exe 同目录或加入 PATH") + } + return nil +} + +func findFFmpeg() (string, error) { + if executable, err := os.Executable(); err == nil { + path := filepath.Join(filepath.Dir(executable), "ffmpeg.exe") + if info, err := os.Stat(path); err == nil && !info.IsDir() { + return path, nil + } + } + return exec.LookPath("ffmpeg") +} + +// Inspect decodes paired frames from the downloaded file. Only clear text cards +// and near-static slide shows are rejected; ambiguous footage is never promoted. +func Inspect(ctx context.Context, path string) (Quality, string, error) { + probe, err := downloader.Probe(ctx, path) + if err != nil { + return QualityUnknown, "完整性校验失败", err + } + ffmpeg, err := findFFmpeg() + if err != nil { + return QualityUnknown, "缺少 ffmpeg", err + } + if probe.Duration < 3 { + return QualityRejected, "视频时长过短", nil + } + var cards, moving, still int + for _, part := range []float64{.1, .3, .5, .7, .9} { + at := math.Min(probe.Duration-1, probe.Duration*part) + a, err := sampleFrame(ctx, ffmpeg, path, at) + if err != nil { + return QualityUnknown, "抽帧失败", err + } + b, err := sampleFrame(ctx, ffmpeg, path, math.Min(probe.Duration-.1, at+.5)) + if err != nil { + return QualityUnknown, "抽帧失败", err + } + if textCard(a) { + cards++ + } + if frameDifference(a, b) > 6 { + moving++ + } else { + still++ + } + } + if cards >= 4 { + return QualityRejected, "多数画面是黑底文字广告", nil + } + if still >= 4 { + return QualityRejected, "多数片段为静止图片", nil + } + if moving >= 2 && cards <= 1 { + return QualityUsable, "检测到连续商品画面变化", nil + } + return QualityUnknown, "画面内容无法可靠判定", nil +} + +func sampleFrame(ctx context.Context, ffmpeg, path string, at float64) (image.Image, error) { + cmd := exec.CommandContext(ctx, ffmpeg, "-hide_banner", "-loglevel", "error", "-ss", strconv.FormatFloat(at, 'f', 2, 64), "-i", path, "-frames:v", "1", "-vf", "scale=96:96", "-f", "image2pipe", "-vcodec", "png", "pipe:1") + cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: windows.CREATE_NO_WINDOW} + raw, err := cmd.Output() + if err != nil { + return nil, err + } + img, _, err := image.Decode(bytes.NewReader(raw)) + return img, err +} + +func textCard(img image.Image) bool { + b := img.Bounds() + total := b.Dx() * b.Dy() + if total == 0 { + return false + } + var dark, white, color int + for y := b.Min.Y; y < b.Max.Y; y++ { + for x := b.Min.X; x < b.Max.X; x++ { + r, g, bl, _ := img.At(x, y).RGBA() + rr, gg, bb := int(r>>8), int(g>>8), int(bl>>8) + if rr < 45 && gg < 45 && bb < 45 { + dark++ + } + if rr > 180 && gg > 180 && bb > 180 { + white++ + } + if max3(rr, gg, bb)-min3(rr, gg, bb) > 35 { + color++ + } + } + } + return float64(dark)/float64(total) > .75 && float64(white)/float64(total) > .008 && float64(color)/float64(total) < .04 +} + +func frameDifference(a, b image.Image) float64 { + ra, rb := a.Bounds(), b.Bounds() + if ra.Dx() != rb.Dx() || ra.Dy() != rb.Dy() { + return 0 + } + var sum float64 + for y := 0; y < ra.Dy(); y++ { + for x := 0; x < ra.Dx(); x++ { + r1, g1, b1, _ := a.At(ra.Min.X+x, ra.Min.Y+y).RGBA() + r2, g2, b2, _ := b.At(rb.Min.X+x, rb.Min.Y+y).RGBA() + sum += math.Abs(float64(int(r1>>8)-int(r2>>8))) + math.Abs(float64(int(g1>>8)-int(g2>>8))) + math.Abs(float64(int(b1>>8)-int(b2>>8))) + } + } + return sum / float64(ra.Dx()*ra.Dy()*3) +} + +func max3(a, b, c int) int { + if a < b { + a = b + } + if a < c { + a = c + } + return a +} +func min3(a, b, c int) int { + if a > b { + a = b + } + if a > c { + a = c + } + return a +} diff --git a/internal/alibaba1688/quality_test.go b/internal/alibaba1688/quality_test.go new file mode 100644 index 0000000..366412a --- /dev/null +++ b/internal/alibaba1688/quality_test.go @@ -0,0 +1,45 @@ +package alibaba1688 + +import ( + "image" + "image/color" + "testing" +) + +func TestTextCardRejectsBlackBackgroundWithWhiteCopy(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 100, 100)) + for y := 0; y < 100; y++ { + for x := 0; x < 100; x++ { + if x >= 20 && x < 80 && y >= 40 && y < 50 { + img.Set(x, y, color.RGBA{255, 255, 255, 255}) + } else { + img.Set(x, y, color.RGBA{0, 0, 0, 255}) + } + } + } + if !textCard(img) { + t.Fatal("black-background sales card should be detected") + } + for y := 0; y < 100; y++ { + for x := 0; x < 100; x++ { + img.Set(x, y, color.RGBA{60, 130, 100, 255}) + } + } + if textCard(img) { + t.Fatal("colored product frame should not be a sales card") + } +} + +func TestFrameDifferenceDetectsContinuousMotion(t *testing.T) { + a := image.NewRGBA(image.Rect(0, 0, 20, 20)) + b := image.NewRGBA(image.Rect(0, 0, 20, 20)) + for y := 0; y < 20; y++ { + for x := 0; x < 20; x++ { + a.Set(x, y, color.RGBA{20, 20, 20, 255}) + b.Set(x, y, color.RGBA{100, 100, 100, 255}) + } + } + if frameDifference(a, a) != 0 || frameDifference(a, b) < 6 { + t.Fatal("motion signal incorrect") + } +} diff --git a/internal/alibaba1688/search.go b/internal/alibaba1688/search.go new file mode 100644 index 0000000..b3e8bf7 --- /dev/null +++ b/internal/alibaba1688/search.go @@ -0,0 +1,292 @@ +package alibaba1688 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "time" + + "cmsp/internal/taobao" +) + +var ErrLoginRequired = errors.New("1688 需要手动登录或完成安全验证") +var offerIDPattern = regexp.MustCompile(`^[0-9]{8,20}$`) + +type target struct { + ID string `json:"id"` + URL string `json:"url"` +} + +func targets(port int) ([]target, error) { + client := http.Client{Timeout: 3 * time.Second} + res, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/json/list", port)) + if err != nil { + return nil, fmt.Errorf("读取 1688 Chrome 页面失败: %w", err) + } + defer res.Body.Close() + var values []target + if err := json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(&values); err != nil { + return nil, err + } + return values, nil +} + +func fetchImage(ctx context.Context, raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil { + return "", fmt.Errorf("商品主图地址无效") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil) + if err != nil { + return "", fmt.Errorf("商品主图地址无效") + } + req.Header.Set("User-Agent", "Mozilla/5.0") + client := http.Client{Timeout: 30 * time.Second} + res, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("下载商品主图失败") + } + defer res.Body.Close() + if res.StatusCode != 200 { + return "", fmt.Errorf("下载商品主图失败：HTTP %d", res.StatusCode) + } + f, err := os.CreateTemp("", "cmsp-1688-image-*.jpg") + if err != nil { + return "", err + } + defer f.Close() + n, err := io.Copy(f, io.LimitReader(res.Body, (10<<20)+1)) + if err != nil || n > 10<<20 { + os.Remove(f.Name()) + return "", fmt.Errorf("商品主图下载失败或超过 10 MB") + } + return f.Name(), nil +} + +// SearchByImage uses the site's visible upload control, then reads public result cards. +// It never reads browser cookies or calls private search endpoints. +func SearchByImage(ctx context.Context, page *taobao.CDP, port int, imageURL string, limit int) ([]string, error) { + image, err := fetchImage(ctx, imageURL) + if err != nil { + return nil, err + } + defer os.Remove(image) + before, err := targets(port) + if err != nil { + return nil, err + } + seen := map[string]bool{} + for _, t := range before { + seen[t.ID] = true + } + if err := page.Navigate("https://www.1688.com/", 3*time.Second); err != nil { + return nil, err + } + if err := checkPage(page, false); err != nil { + return nil, err + } + var input struct { + NodeID int `json:"nodeId"` + } + for i := 0; i < 10 && input.NodeID == 0; i++ { + raw, err := page.Send("DOM.getDocument", map[string]any{"depth": 1}) + if err != nil { + return nil, err + } + var doc struct { + Root struct { + NodeID int `json:"nodeId"` + } `json:"root"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, err + } + raw, err = page.Send("DOM.querySelector", map[string]any{"nodeId": doc.Root.NodeID, "selector": `input[type="file"][accept*="jpg"]`}) + if err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &input); err != nil { + return nil, err + } + if input.NodeID == 0 { + if err := wait(ctx, time.Second); err != nil { + return nil, err + } + } + } + if input.NodeID == 0 { + return nil, fmt.Errorf("1688 以图搜上传框未找到") + } + if _, err := page.Send("DOM.setFileInputFiles", map[string]any{"nodeId": input.NodeID, "files": []string{image}}); err != nil { + return nil, err + } + var button string + for i := 0; i < 10; i++ { + if err := wait(ctx, time.Second); err != nil { + return nil, err + } + button, err = page.Evaluate(`(()=>{const b=[...document.querySelectorAll('button')].find(x=>x.textContent.trim()==='搜索'&&x.closest('[class*=imageSearch]'));if(!b||b.disabled||!b.closest('[class*=imageSearch]').innerText.includes('已上传1张图'))return '';const r=b.getBoundingClientRect();return JSON.stringify({x:r.x+r.width/2,y:r.y+r.height/2})})()`) + if err != nil { + return nil, err + } + if button != "" { + break + } + } + if button == "" { + return nil, fmt.Errorf("1688 以图搜图片未完成上传") + } + // The preview becomes visible before the site has finished preparing its search image. + if err := wait(ctx, 2*time.Second); err != nil { + return nil, err + } + var pos struct{ X, Y float64 } + if err := json.Unmarshal([]byte(button), &pos); err != nil { + return nil, err + } + if _, err := page.Send("Page.bringToFront", nil); err != nil { + return nil, err + } + for _, typ := range []string{"mouseMoved", "mousePressed", "mouseReleased"} { + if _, err := page.Send("Input.dispatchMouseEvent", map[string]any{"type": typ, "x": pos.X, "y": pos.Y, "button": "left", "clickCount": 1}); err != nil { + return nil, err + } + if typ == "mousePressed" { + if err := wait(ctx, 100*time.Millisecond); err != nil { + return nil, err + } + } + } + var resultURL, resultID string + for i := 0; i < 15 && resultURL == ""; i++ { + if err := wait(ctx, time.Second); err != nil { + return nil, err + } + list, err := targets(port) + if err != nil { + return nil, err + } + for _, t := range list { + if !seen[t.ID] && strings.HasPrefix(t.URL, "https://air.1688.com/kapp/1688-search/pc-image-search/") { + resultURL, resultID = t.URL, t.ID + break + } + } + } + if resultURL == "" { + return nil, fmt.Errorf("1688 图搜未打开结果页") + } + if err := page.Navigate(resultURL, 3*time.Second); err != nil { + return nil, err + } + if resultID != "" { + client := http.Client{Timeout: 2 * time.Second} + if response, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/json/close/%s", port, resultID)); err == nil { + response.Body.Close() + } + } + if err := checkPage(page, false); err != nil { + return nil, err + } + if limit < 10 { + limit = 10 + } + expr := `JSON.stringify([...new Set([...document.querySelectorAll('[data-extra]')].map(x=>{try{return String(JSON.parse(x.dataset.extra).offerId||'')}catch(e){return ''}}).filter(Boolean))].slice(0,` + strconv.Itoa(limit) + `))` + var offers []string + for i := 0; i < 8; i++ { + text, err := page.Evaluate(expr) + if err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(text), &offers); err != nil { + return nil, err + } + if len(offers) > 0 { + break + } + if err := wait(ctx, time.Second); err != nil { + return nil, err + } + } + valid := offers[:0] + for _, id := range offers { + if offerIDPattern.MatchString(id) { + valid = append(valid, id) + } + } + return valid, nil +} + +func OfferVideo(page *taobao.CDP, id string, delay time.Duration) (string, error) { + if !offerIDPattern.MatchString(id) { + return "", fmt.Errorf("1688 商品 ID 无效") + } + if err := page.Navigate("https://detail.1688.com/offer/"+id+".html", delay); err != nil { + return "", err + } + if err := checkPage(page, true); err != nil { + return "", err + } + text, err := page.Evaluate(`JSON.stringify([...document.querySelectorAll('video')].map(v=>v.currentSrc||v.src||'').filter(Boolean))`) + if err != nil { + return "", err + } + var values []string + if err := json.Unmarshal([]byte(text), &values); err != nil { + return "", err + } + for _, value := range values { + u, err := url.Parse(value) + if err != nil || u.Scheme != "https" || u.User != nil { + continue + } + host := strings.ToLower(u.Hostname()) + if host == "cloud.video.taobao.com" || strings.HasSuffix(host, ".cloudvideocdn.com") { + return value, nil + } + } + return "", nil +} + +func checkPage(page *taobao.CDP, detail bool) error { + raw, err := page.Evaluate(`JSON.stringify({url:location.href,title:document.title,text:(document.body?.innerText||'').slice(0,1200)})`) + if err != nil { + return err + } + var p struct{ URL, Title, Text string } + if err := json.Unmarshal([]byte(raw), &p); err != nil { + return err + } + u, err := url.Parse(p.URL) + if err != nil || !strings.HasSuffix(strings.ToLower(u.Hostname()), "1688.com") { + return ErrLoginRequired + } + if detail && (u.Hostname() != "detail.1688.com" || !strings.HasPrefix(u.Path, "/offer/")) { + return ErrLoginRequired + } + for _, word := range []string{"访问被拒绝", "安全验证", "请输入验证码", "请先登录"} { + if strings.Contains(p.Title, word) || strings.Contains(p.Text, word) { + return ErrLoginRequired + } + } + return nil +} + +func wait(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} diff --git a/internal/config/config.go b/internal/config/config.go index b0344f5..043b093 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -35,11 +35,12 @@ import ( // yaml tag 就是配置文件里的键名。改名会导致老配置读不出来， // 所以除非有充分理由，不要改已有字段的 tag。 type Config struct { - ERPGo ERPGoConfig `yaml:"erpgo" json:"erpgo"` - Huohanhan HuohanhanConfig `yaml:"huohanhan" json:"huohanhan"` - Taobao TaobaoConfig `yaml:"taobao" json:"taobao"` - Download DownloadConfig `yaml:"download" json:"download"` - Upload UploadConfig `yaml:"upload" json:"upload"` + ERPGo ERPGoConfig `yaml:"erpgo" json:"erpgo"` + Huohanhan HuohanhanConfig `yaml:"huohanhan" json:"huohanhan"` + Taobao TaobaoConfig `yaml:"taobao" json:"taobao"` + Alibaba1688 TaobaoConfig `yaml:"alibaba1688" json:"alibaba1688"` + Download DownloadConfig `yaml:"download" json:"download"` + Upload UploadConfig `yaml:"upload" json:"upload"` } // ERPGoConfig 用于查询店铺和商品；APIKey 只保存在本机，禁止进入日志。 @@ -90,6 +91,7 @@ type TaobaoConfig struct { // DownloadConfig 是下载与任务参数。 type DownloadConfig struct { + Source string `yaml:"source" json:"source"` // VideoDir 是下载的视频保存目录。 VideoDir string `yaml:"video_dir" json:"videoDir"` // MaxVideosPerProduct 是每个商品最多下载几个视频。 @@ -131,7 +133,14 @@ func Default() Config { DebugPortStart: 19666, DebugPortEnd: 19765, }, + Alibaba1688: TaobaoConfig{ + ChromePath: `C:\Program Files\Google\Chrome\Application\chrome.exe`, + UserDataDir: filepath.Join(os.Getenv("LOCALAPPDATA"), "cmsp", "1688浏览器"), + DebugPortStart: 19766, + DebugPortEnd: 19865, + }, Download: DownloadConfig{ + Source: "taobao", VideoDir: DefaultVideoDir(), MaxVideosPerProduct: 3, SearchTopN: 5, @@ -195,6 +204,24 @@ func (c Config) Validate() error { } // 旧 huohanhan 配置只为兼容现有 config.yaml 保留，不再用于请求。 t := c.Taobao + if c.Download.Source != "taobao" && c.Download.Source != "1688" { + return fmt.Errorf("下载来源须为 taobao 或 1688") + } + if strings.TrimSpace(c.Alibaba1688.ChromePath) == "" || strings.TrimSpace(c.Alibaba1688.UserDataDir) == "" { + return fmt.Errorf("1688 专属 Chrome 路径和用户数据目录不能为空") + } + if filepath.Clean(c.Alibaba1688.UserDataDir) == filepath.Clean(t.UserDataDir) { + return fmt.Errorf("1688 和淘宝不能共用 Chrome 用户数据目录") + } + if err := checkPort(c.Alibaba1688.DebugPortStart, "1688 调试端口起始"); err != nil { + return err + } + if err := checkPort(c.Alibaba1688.DebugPortEnd, "1688 调试端口结束"); err != nil { + return err + } + if c.Alibaba1688.DebugPortEnd < c.Alibaba1688.DebugPortStart { + return fmt.Errorf("1688 调试端口结束不能小于起始端口") + } if strings.TrimSpace(t.ChromePath) == "" { return fmt.Errorf("Chrome 可执行文件路径不能为空") } @@ -375,7 +402,16 @@ taobao: debug_port_start: %d debug_port_end: %d +alibaba1688: + # 1688 使用独立 Profile，登录只能在专属 Chrome 中手动完成。 + chrome_path: %s + user_data_dir: %s + debug_port_start: %d + debug_port_end: %d + download: + # 下载来源：taobao 或 1688；仅影响新批次。 + source: %s # 下载的视频保存到哪个目录。 video_dir: %s @@ -421,6 +457,11 @@ upload: quoted(c.Taobao.UserDataDir), c.Taobao.DebugPortStart, c.Taobao.DebugPortEnd, + quoted(c.Alibaba1688.ChromePath), + quoted(c.Alibaba1688.UserDataDir), + c.Alibaba1688.DebugPortStart, + c.Alibaba1688.DebugPortEnd, + quoted(c.Download.Source), quoted(c.Download.VideoDir), c.Download.MaxVideosPerProduct, c.Download.SearchTopN, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7a16aa0..7c8c48a 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -53,6 +53,26 @@ func TestDefaultConfigIsValid(t *testing.T) { } } +func TestVideoSourcePersistsAndKeepsIndependentBrowser(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + cfg := Default() + cfg.Download.Source = "1688" + if cfg.Alibaba1688.UserDataDir == cfg.Taobao.UserDataDir { + t.Fatal("1688 and Taobao must not share Chrome profile") + } + if err := Save(path, cfg); err != nil { + t.Fatal(err) + } + loaded, err := Load(path) + if err != nil || loaded.Download.Source != "1688" || loaded.Alibaba1688.UserDataDir != cfg.Alibaba1688.UserDataDir { + t.Fatalf("source/browser config not persisted: %v", err) + } + cfg.Download.Source = "unknown" + if cfg.Validate() == nil { + t.Fatal("unsupported source accepted") + } +} + func TestValidateRejectsBadValues(t *testing.T) { // 每个用例只改一个字段，这样失败时能一眼看出是哪条规则出问题。 cases := []struct { diff --git a/internal/store/video.go b/internal/store/video.go index 1ce31bd..cced848 100644 --- a/internal/store/video.go +++ b/internal/store/video.go @@ -1,6 +1,7 @@ package store import ( + "database/sql" "fmt" "os" "strings" @@ -34,6 +35,39 @@ func (s *Store) ReplaceVideos(productID string, items []Video, now string) error return fmt.Errorf("开启视频替换事务失败：%w", err) } defer tx.Rollback() + if err := replaceVideosTx(tx, productID, items, now); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("提交视频替换事务失败：%w", err) + } + return nil +} + +// CompleteVideoDownload commits the accepted video and product status together. +func (s *Store) CompleteVideoDownload(productID string, video Video) error { + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("开启视频下载完成事务失败：%w", err) + } + defer tx.Rollback() + if err := replaceVideosTx(tx, productID, []Video{video}, video.CreatedAt); err != nil { + return err + } + result, err := tx.Exec(`UPDATE products SET video_status = ?, download_status = ?, last_error = '' WHERE id = ?`, VideoFound, DownloadDone, productID) + if err != nil { + return fmt.Errorf("更新商品下载状态失败：%w", err) + } + if count, err := result.RowsAffected(); err != nil || count != 1 { + return fmt.Errorf("商品下载状态更新数量异常：%d，%v", count, err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("提交视频下载完成事务失败：%w", err) + } + return nil +} + +func replaceVideosTx(tx *sql.Tx, productID string, items []Video, now string) error { if _, err := tx.Exec(`DELETE FROM videos WHERE product_id = ?`, productID); err != nil { return fmt.Errorf("清理商品旧视频失败：%w", err) } @@ -56,9 +90,6 @@ func (s *Store) ReplaceVideos(productID string, items []Video, now string) error return fmt.Errorf("写入商品视频失败：%w", err) } } - if err := tx.Commit(); err != nil { - return fmt.Errorf("提交视频替换事务失败：%w", err) - } return nil } diff --git a/internal/store/video_test.go b/internal/store/video_test.go index d73a776..c6b813f 100644 --- a/internal/store/video_test.go +++ b/internal/store/video_test.go @@ -6,6 +6,31 @@ import ( "testing" ) +func TestCompleteVideoDownloadCommitsVideoAndStatusTogether(t *testing.T) { + s := newTestStore(t) + if _, err := s.DB().Exec(`INSERT INTO products (id, item_id, video_status, download_status) VALUES ('p1', 'fictional-1', 'pending', 'pending')`); err != nil { + t.Fatal(err) + } + video := Video{SourceItem: "fictional-1688-1", LocalPath: "D:/fictional.mp4", Status: VideoStatusDownloaded, CreatedAt: "2026-10-07 12:00:00"} + if err := s.CompleteVideoDownload("missing", video); err == nil { + t.Fatal("missing product must roll back") + } + if items, err := s.ListVideos("missing"); err != nil || len(items) != 0 { + t.Fatalf("rollback failed: %v %+v", err, items) + } + if err := s.CompleteVideoDownload("p1", video); err != nil { + t.Fatal(err) + } + product, found, err := s.GetProduct("p1") + if err != nil || !found || product.VideoStatus != VideoFound || product.DownloadStatus != DownloadDone { + t.Fatalf("status mismatch: %+v %v %v", product, found, err) + } + items, err := s.ListVideos("p1") + if err != nil || len(items) != 1 || items[0].SourceItem != video.SourceItem { + t.Fatalf("video mismatch: %+v %v", items, err) + } +} + func TestReplaceVideos按商品全量替换(t *testing.T) { s := newTestStore(t) first := []Video{ diff --git a/internal/taobao/chrome.go b/internal/taobao/chrome.go index 56435af..2d46364 100644 --- a/internal/taobao/chrome.go +++ b/internal/taobao/chrome.go @@ -35,6 +35,11 @@ type BrowserState struct { // EnsureBrowser 复用归属明确且可连接的专属 Chrome；否则安全地启动一个新实例。 func EnsureBrowser(ctx context.Context, cfg config.TaobaoConfig, openURL string) (BrowserState, error) { + return EnsureBrowserWithState(ctx, cfg, openURL, browserStateName) +} + +// EnsureBrowserWithState isolates runtime ownership metadata for another site/profile. +func EnsureBrowserWithState(ctx context.Context, cfg config.TaobaoConfig, openURL, stateName string) (BrowserState, error) { if ctx == nil { ctx = context.Background() } @@ -51,7 +56,10 @@ func EnsureBrowser(ctx context.Context, cfg config.TaobaoConfig, openURL string) return BrowserState{}, fmt.Errorf("创建 Chrome 用户数据目录失败：%w", err) } - stateFile := config.ResolveDataPath(browserStateName) + if stateName != browserStateName && stateName != "1688浏览器运行状态.json" { + return BrowserState{}, fmt.Errorf("不支持的浏览器状态文件") + } + stateFile := config.ResolveDataPath(stateName) state, found, readErr := readBrowserState(stateFile) if readErr == nil && found && browserStateReusable(state, profile) { return state, nil diff --git a/internal/task/task.go b/internal/task/task.go index 64bf2ac..a1d718c 100644 --- a/internal/task/task.go +++ b/internal/task/task.go @@ -264,6 +264,14 @@ func (r *Runner) run(ctx context.Context, queue []queuedProduct) { r.completeProduct(0, err == nil, err != nil) } else { downloaded, downloadErr := RunDownloads(ctx, r.options.Concurrency, work) + if errors.Is(downloadErr, ErrLoginRequired) || errors.Is(downloadErr, ErrRiskBlocked) { + r.setStopReason("login_required") + if errors.Is(downloadErr, ErrRiskBlocked) { + r.setStopReason(StopReasonRiskBlocked) + } + r.finish(StateLoginRequired) + return + } r.completeProduct(downloaded, false, downloadErr != nil) } }