Merge branch 'feat/37-1688-video-check' into integrate/37

This commit is contained in:
ila
2026-10-07 15:59:30 +08:00
19 changed files with 920 additions and 19 deletions
+32
View File
@@ -15,6 +15,7 @@ import (
"sync"
"time"
"cmsp/internal/alibaba1688"
"cmsp/internal/config"
"cmsp/internal/downloader"
"cmsp/internal/erpgo"
@@ -145,6 +146,13 @@ func (a *App) beforeClose(ctx context.Context) bool {
} else if closed {
a.log.Info("退出前已关闭淘宝专属 Chrome")
}
stateFile = config.ResolveDataPath("1688浏览器运行状态.json")
closed, err = taobao.CloseBrowserOnExit(stateFile, a.cfg.Alibaba1688.UserDataDir)
if err != nil {
a.log.Warn("退出时保留未能确认归属或关闭的 1688 专属 Chrome:%v", err)
} else if closed {
a.log.Info("退出前已关闭 1688 专属 Chrome")
}
return false
}
@@ -432,6 +440,20 @@ func (a *App) FetchVideosForProduct(productID string) (FetchResult, error) {
return FetchResult{ProductID: productID}, err
}
ctx := a.appContext()
if a.cfg.Download.Source == "1688" {
if err := alibaba1688.CheckFFmpeg(); err != nil {
return FetchResult{ProductID: productID}, err
}
result, work, err := a.prepare1688Video(ctx, productID, a.cfg)
if err != nil {
return result, err
}
if work.Skipped {
return result, nil
}
result.DownloadedCount, err = task.RunDownloads(ctx, 1, work)
return result, err
}
riskGuard := task.NewEmptyRiskGuard(a.cfg.Download.RiskEmptyThreshold)
result, work, err := a.prepareVideoFetch(ctx, productID, riskGuard, make(taobao.DetailCache), &taobao.LoginGuard{})
if err != nil {
@@ -917,6 +939,12 @@ func (a *App) StartVideoTask(productIDs []string) error {
if err := downloader.CheckFFProbe(); err != nil {
return err
}
batchConfig := a.cfg
if batchConfig.Download.Source == "1688" {
if err := alibaba1688.CheckFFmpeg(); err != nil {
return err
}
}
a.videoTaskMu.Lock()
defer a.videoTaskMu.Unlock()
@@ -948,6 +976,10 @@ func (a *App) StartVideoTask(productIDs []string) error {
}, nil
},
Prepare: func(ctx context.Context, product task.Product) (task.Work, error) {
if batchConfig.Download.Source == "1688" {
_, work, err := a.prepare1688Video(ctx, product.ID, batchConfig)
return work, err
}
_, work, err := a.prepareVideoFetch(ctx, product.ID, riskGuard, detailCache, loginGuard)
return work, err
},
+148
View File
@@ -0,0 +1,148 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"cmsp/internal/alibaba1688"
"cmsp/internal/config"
"cmsp/internal/downloader"
"cmsp/internal/store"
"cmsp/internal/taobao"
"cmsp/internal/task"
)
// Open1688Login opens an isolated profile. The operator performs login manually.
func (a *App) Open1688Login() error {
state, err := taobao.EnsureBrowserWithState(a.appContext(), a.cfg.Alibaba1688, "https://www.1688.com/", "1688浏览器运行状态.json")
if err != nil {
return err
}
page, err := taobao.Connect(state.Port)
if err != nil {
return err
}
defer page.Close()
if err := page.Navigate("https://www.1688.com/", 0); err != nil {
return err
}
a.log.Info("已打开 1688 专属 Chrome,请在浏览器中手动完成登录")
return nil
}
func (a *App) prepare1688Video(ctx context.Context, productID string, cfg config.Config) (FetchResult, task.Work, error) {
result := FetchResult{ProductID: productID}
product, found, err := a.db.GetProduct(productID)
if err != nil {
return result, task.Work{}, err
}
if !found {
return result, task.Work{}, fmt.Errorf("找不到商品:%s", productID)
}
if product.DownloadStatus == store.DownloadDone {
return result, task.Work{Skipped: true}, nil
}
if product.MainImage == "" {
return result, task.Work{}, fmt.Errorf("商品没有可用于图搜的主图")
}
state, err := taobao.EnsureBrowserWithState(a.appContext(), cfg.Alibaba1688, "https://www.1688.com/", "1688浏览器运行状态.json")
if err != nil {
return result, task.Work{}, err
}
page, err := taobao.Connect(state.Port)
if err != nil {
return result, task.Work{}, err
}
offers, err := alibaba1688.SearchByImage(ctx, page, state.Port, product.MainImage, cfg.Download.SearchTopN)
if err != nil {
page.Close()
if errors.Is(err, alibaba1688.ErrLoginRequired) {
return result, task.Work{}, fmt.Errorf("%w:%v", task.ErrLoginRequired, err)
}
return result, task.Work{}, err
}
if len(offers) == 0 {
page.Close()
return result, task.Work{Skipped: true}, nil
}
root, err := filepath.Abs(cfg.Download.VideoDir)
if err != nil {
page.Close()
return result, task.Work{}, err
}
productDir := filepath.Join(root, downloader.SafeDirName(product.ItemID))
if err := os.MkdirAll(productDir, 0755); err != nil {
page.Close()
return result, task.Work{}, err
}
dl := downloader.New()
var record store.Video
var target string
work := task.Work{Downloads: []task.DownloadFunc{func(downloadCtx context.Context) error {
defer page.Close()
for i, id := range offers {
if err := downloadCtx.Err(); err != nil {
return err
}
videoURL, err := alibaba1688.OfferVideo(page, id, time.Duration(cfg.Download.DetailWaitSeconds*float64(time.Second)))
if err != nil {
if errors.Is(err, alibaba1688.ErrLoginRequired) {
return fmt.Errorf("%w:%v", task.ErrLoginRequired, err)
}
a.log.Warn("蝦皮商品 %s 的 1688 候选 %s 打开失败:%v", product.ItemID, id, err)
continue
}
if videoURL == "" {
a.log.Info("蝦皮商品 %s 的 1688 候选 %s 没有可用视频", product.ItemID, id)
continue
}
f, err := os.CreateTemp(productDir, ".cmsp-1688-*.mp4")
if err != nil {
return err
}
temp := f.Name()
f.Close()
os.Remove(temp)
res, err := dl.Download(downloadCtx, videoURL, "https://detail.1688.com/offer/"+id+".html", temp, cfg.Download.DownloadRetries)
if err != nil {
os.Remove(temp)
a.log.Warn("蝦皮商品 %s 的 1688 候选 %s 视频下载失败:%v", product.ItemID, id, err)
continue
}
quality, reason, err := alibaba1688.Inspect(downloadCtx, temp)
if err != nil || quality != alibaba1688.QualityUsable {
os.Remove(temp)
a.log.Warn("蝦皮商品 %s 的 1688 候选 %s 已跳过(%s,%s)", product.ItemID, id, quality, reason)
continue
}
target = filepath.Join(productDir, downloader.Filename(product.ItemID, 1))
if _, err := os.Stat(target); err == nil {
os.Remove(temp)
return fmt.Errorf("目标视频文件已存在,保留原文件,未覆盖")
}
if err := os.Rename(temp, target); err != nil {
os.Remove(temp)
return err
}
record = store.Video{SourceItem: id, SourceURL: videoURL, LocalPath: target, FileSize: res.Size, Status: store.VideoStatusDownloaded, CreatedAt: time.Now().Format("2006-01-02 15:04:05")}
a.log.Success("蝦皮商品 %s 的 1688 候选 %d/%d 通过逐条检测并下载", product.ItemID, i+1, len(offers))
return nil
}
return fmt.Errorf("1688 候选均无可用商品视频,原任务状态已保留")
}}}
work.Finalize = func(errs []error) error {
if len(errs) == 0 || errs[0] != nil {
return nil
}
if err := a.db.CompleteVideoDownload(product.ID, record); err != nil {
os.Remove(target)
return err
}
return nil
}
return result, work, nil
}
+9
View File
@@ -55,7 +55,16 @@ taobao:
debug_port_start: 19666
debug_port_end: 19765
alibaba1688:
# 1688 单独的 Chrome Profile;在专属浏览器中手动登录。
chrome_path: "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
user_data_dir: "C:\\Users\\你的用户名\\AppData\\Local\\cmsp\\1688浏览器"
debug_port_start: 19766
debug_port_end: 19865
download:
# taobao 或 1688;切换仅影响新启动的下载批次。
source: taobao
# 下载的视频保存到哪个目录。
video_dir: "D:\\chengma\\cmsp\\运行数据\\视频"
+6 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Architecture-and-Code-Map
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Architecture-and-Code-Map.-
wiki_revision: 9ebe51001ec9054656478312baedb3387d172c7a
synchronized_at: 2026-10-05T10:01:39Z
wiki_revision: 0ba3b5f3e6cf4630ef6fde05b0b17ba27a00b50b
synchronized_at: 2026-10-07T07:49:50Z
<!-- gitea-wiki-mirror:end -->
# 架构与代码地图
@@ -180,3 +180,7 @@ cmsp/
- **凭据不进入仓库。** 账号、密码、token、Cookie、Chrome Profile 内容不得出现在代码、日志、工单、Wiki 和提交中。
- **SQLite 是任务状态的唯一事实来源。** 不允许再用 JSON 文件维护第二份任务状态。
- **前端不做安全判断。** 按钮禁用只是提示,真正的校验必须在 Go 侧执行。
## 1688 视频下载来源(2026-10-07,#37)
商品列表“下载视频”旁选择淘宝或 1688;`download.source` 写入本机 `config.yaml`,批次启动时固定来源。1688 使用独立 Chrome Profile,操作者手动登录。`app_1688.go` 调用 `internal/alibaba1688/search.go`:上传商品主图到网页以图搜,按结果顺序进入候选详情,读取页面视频地址。`internal/downloader` 将候选 MP4 下载到临时文件;`internal/alibaba1688/quality.go` 用 ffprobe 校验、ffmpeg 抽帧检测黑底文字广告和静止拼图。只有判为 `usable` 的视频才晋升为商品文件,`internal/store.CompleteVideoDownload` 在同一 SQLite 事务中写视频记录与商品已下载状态。拒绝、无法判定、候选下载失败时保留原商品状态;上传仍沿用原有 erpgo 流程。
+6 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Business-Rules-and-Glossary
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Business-Rules-and-Glossary.-
wiki_revision: 6deec49c2ba48aaf96a5842b4d525352afd6cf87
synchronized_at: 2026-10-06T07:37:40Z
wiki_revision: e05847d648d87a01f714ca7cb0af177e7630dcdf
synchronized_at: 2026-10-07T07:49:50Z
<!-- gitea-wiki-mirror:end -->
# 业务规则与术语
@@ -149,3 +149,7 @@ MTOP 搜索方式、签名与请求参数保持现状。商品原链接以实际
- Shopee 侧的实际生效延迟与验证方式。
每确认一项,就更新本页对应规则并删除此处条目;确认过程记录在对应工单中。
## 1688 作为可选视频来源(2026-10-07,#37)
下载来源由使用者在商品列表下载按钮旁选择并保存在本机配置,仅影响新批次。1688 与淘宝使用不同 Chrome Profile,登录由使用者手动完成。1688 图搜按结果顺序查找第一个有视频且通过内容检测的候选;每个候选先下载到临时文件,再检查时长、画面连续变化、黑底文字广告与静止图片。检测失败或无法可靠判定时删除临时文件,不写“已下载”;全部候选无可用视频时保留原状态,允许重试或换来源。通过检测后写入原有“已下载/待上传”状态,上传确认与 erpgo 上传流程不变。自动检测不能证明候选与目标商品完全同款,也不能证明视频使用权;批量上传的现有确认行为保持不变。
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Local-Development-and-Verification
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Local-Development-and-Verification.-
wiki_revision: 697d2e353c3b4b3f6648b7a08e4752235c42c7e7
synchronized_at: 2026-10-06T07:37:41Z
wiki_revision: 8c23a3a5b52f132282e9b6b19ae08b6fbb879c55
synchronized_at: 2026-10-07T07:49:51Z
<!-- gitea-wiki-mirror:end -->
# 本地开发与验证
@@ -232,3 +232,7 @@ Go 实现完成后适用:
6. 有工单的任务在工单追加最终证据评论,状态置为「待验收」,等待人工验收。
涉及货憨憨写操作、账号凭据、SQLite 结构变更、并发或删除数据时,必须先确认再执行,不能靠试错获得风险反馈。
## 1688 视频来源运行要求(2026-10-07,#37)
在商品列表下载按钮旁选“1688”,参数设置中检查 `alibaba1688.chrome_path`、独立 `user_data_dir` 与调试端口范围,点击“打开 1688 登录”并手动登录。1688 视频内容检测除了既有 ffprobe,还需要 `ffmpeg.exe`;打包运行时把二者放在 `cmsp.exe` 同目录,或加入 PATH。缺少 ffmpeg 时启动 1688 下载前即提示,不改变淘宝下载。`download.source` 保存在本机 `config.yaml`,默认 `taobao`。验证命令:`go test ./...`,`wails build`;真机验收分别用广告文字候选和正常商品视频检查拒绝、放行及 SQLite 状态。
+6 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Troubleshooting
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Troubleshooting
wiki_revision: 0a176fa4cf7f3bd6a2c6b11ae3c6a58453500c85
synchronized_at: 2026-10-05T02:28:58Z
wiki_revision: 7b2d3a3a39b3fa1b8b90f564e6ad5c727771dad7
synchronized_at: 2026-10-07T07:49:52Z
<!-- gitea-wiki-mirror:end -->
# 故障排查
@@ -142,3 +142,7 @@ SYNC_IN_PROGRESS 表示已有商品同步,请等待结束;Go侧拒绝重叠
- 无法判断改动风险等级。
停止不等于失败。把现象、已确认事实和不确定的部分写清楚,比继续试更有价值。
## 1688 图搜与视频内容检测(2026-10-07,#37)
“1688 以图搜上传框未找到”或“图搜未打开结果页”时,先在独立 Chrome 中确认手动登录、首页正常显示和网页以图搜可用;不要自动绕过登录或安全验证。页面刚加载时控件和图片处理可能尚未就绪,程序会等待后再点击;持续失败时保留原任务状态,人工检查站点页面是否改版或访问受限。“1688 内容检测需要 ffmpeg.exe”时把 ffmpeg.exe 放在 cmsp.exe 同目录或加入 PATH,并保留既有 ffprobe。“候选均无可用商品视频”表示所查候选没有通过检测的 MP4,不表示该蝦皮商品在所有来源均无视频;可手动切换来源后重试。
+32 -1
View File
@@ -62,6 +62,28 @@ const emptyQuery = () => ({
pageSize: 20,
})
const query = ref(emptyQuery())
const videoSource = ref('taobao')
const sourceSaving = ref(false)
const videoSourceOptions = [
{ label: '淘宝', value: 'taobao' },
{ label: '1688', value: '1688' },
]
async function changeVideoSource(value) {
if (taskRunning.value || sourceSaving.value) return
sourceSaving.value = true
try {
const cfg = await AppBindings.GetConfig()
cfg.download.source = value
await AppBindings.SaveConfig(cfg)
videoSource.value = value
message.info(`新下载批次将使用${value === '1688' ? '1688' : '淘宝'}`)
} catch (err) {
message.error(`保存下载来源失败:${err}`)
} finally {
sourceSaving.value = false
}
}
// 商品状态取值来自货憨憨接口文档,不要随便加。
const statusOptions = [
@@ -770,7 +792,7 @@ async function startVideoTask() {
await StartVideoTask(selectedIDs)
await refreshTaskProgress()
startTaskPolling()
message.info('批量取视频已启动')
message.info(`批量取视频已启动,来源:${videoSource.value === '1688' ? '1688' : '淘宝'}`)
} catch (err) {
message.error(`启动批量取视频失败:${err}`)
}
@@ -882,6 +904,7 @@ async function onTaskLoginRequired(progress) {
}
onMounted(() => {
AppBindings.GetConfig().then(cfg => { videoSource.value = cfg.download?.source || 'taobao' })
// 先监听再读缓存:后台刷新若恰好此时完成,不会漏掉最新列表。
EventsOn('shops:refreshed', onShopsRefreshed)
EventsOn('shops:refresh-failed', onShopsRefreshFailed)
@@ -976,6 +999,14 @@ onUnmounted(() => {
>
下载数据
</n-button>
<span class="lb">视频来源</span>
<n-select
:value="videoSource"
:options="videoSourceOptions"
:disabled="taskRunning || running !== '' || sourceSaving"
style="width: 115px"
@update:value="changeVideoSource"
/>
<n-button
:loading="taskRunning"
:disabled="taskRunning || running !== ''"
+34
View File
@@ -28,6 +28,7 @@ const dialog = useDialog()
const cfg = ref(null)
const saving = ref(false)
const openingTaobao = ref(false)
const opening1688 = ref(false)
const checkingTaobao = ref(false)
const taobaoStatus = ref(null)
const resettingNoneProducts = ref(false)
@@ -111,6 +112,19 @@ async function pickDir(field, title) {
if (!picked) return
if (field === 'userDataDir') cfg.value.taobao.userDataDir = picked
if (field === 'videoDir') cfg.value.download.videoDir = picked
if (field === 'alibaba1688UserDataDir') cfg.value.alibaba1688.userDataDir = picked
}
async function open1688Login() {
opening1688.value = true
try {
await AppBindings.Open1688Login()
message.info('已打开 1688 专属 Chrome,请手动完成登录')
} catch (err) {
message.error(`打开 1688 失败:${err}`)
} finally {
opening1688.value = false
}
}
async function openTaobaoLogin() {
@@ -272,6 +286,26 @@ onUnmounted(() => {
</n-alert>
</n-card>
<n-card title="1688 专属浏览器" size="small">
<n-form label-placement="top" size="small">
<n-form-item label="Chrome 可执行文件">
<n-input v-model:value="cfg.alibaba1688.chromePath" />
</n-form-item>
<n-form-item label="独立用户数据目录(登录由你手动完成)">
<n-input-group>
<n-input v-model:value="cfg.alibaba1688.userDataDir" />
<n-button @click="pickDir('alibaba1688UserDataDir', '选择 1688 Chrome 用户数据目录')">选择目录</n-button>
</n-input-group>
</n-form-item>
<n-form-item label="调试端口范围">
<n-input-number v-model:value="cfg.alibaba1688.debugPortStart" :show-button="false" style="width: 96px" />
<span class="dash">—</span>
<n-input-number v-model:value="cfg.alibaba1688.debugPortEnd" :show-button="false" style="width: 96px" />
</n-form-item>
</n-form>
<n-button :loading="opening1688" @click="open1688Login">打开 1688 登录</n-button>
</n-card>
<!-- 下载与任务 -->
<n-card title="下载与任务" size="small">
<n-form label-placement="top" size="small">
+1 -1
View File
@@ -5,6 +5,7 @@ go 1.25.0
require (
github.com/gorilla/websocket v1.5.3
github.com/wailsapp/wails/v2 v2.15.0
golang.org/x/sys v0.47.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.57.0
)
@@ -38,7 +39,6 @@ require (
github.com/wailsapp/mimetype v1.4.1 // indirect
golang.org/x/crypto v0.53.0 // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.39.0 // indirect
modernc.org/libc v1.74.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
+161
View File
@@ -0,0 +1,161 @@
package alibaba1688
import (
"bytes"
"context"
"fmt"
"image"
_ "image/png"
"math"
"os"
"os/exec"
"path/filepath"
"strconv"
"syscall"
"cmsp/internal/downloader"
"golang.org/x/sys/windows"
)
type Quality string
const (
QualityUsable Quality = "usable"
QualityRejected Quality = "rejected"
QualityUnknown Quality = "unknown"
)
func CheckFFmpeg() error {
_, err := findFFmpeg()
if err != nil {
return fmt.Errorf("1688 内容检测需要 ffmpeg.exe:请放在 cmsp.exe 同目录或加入 PATH")
}
return nil
}
func findFFmpeg() (string, error) {
if executable, err := os.Executable(); err == nil {
path := filepath.Join(filepath.Dir(executable), "ffmpeg.exe")
if info, err := os.Stat(path); err == nil && !info.IsDir() {
return path, nil
}
}
return exec.LookPath("ffmpeg")
}
// Inspect decodes paired frames from the downloaded file. Only clear text cards
// and near-static slide shows are rejected; ambiguous footage is never promoted.
func Inspect(ctx context.Context, path string) (Quality, string, error) {
probe, err := downloader.Probe(ctx, path)
if err != nil {
return QualityUnknown, "完整性校验失败", err
}
ffmpeg, err := findFFmpeg()
if err != nil {
return QualityUnknown, "缺少 ffmpeg", err
}
if probe.Duration < 3 {
return QualityRejected, "视频时长过短", nil
}
var cards, moving, still int
for _, part := range []float64{.1, .3, .5, .7, .9} {
at := math.Min(probe.Duration-1, probe.Duration*part)
a, err := sampleFrame(ctx, ffmpeg, path, at)
if err != nil {
return QualityUnknown, "抽帧失败", err
}
b, err := sampleFrame(ctx, ffmpeg, path, math.Min(probe.Duration-.1, at+.5))
if err != nil {
return QualityUnknown, "抽帧失败", err
}
if textCard(a) {
cards++
}
if frameDifference(a, b) > 6 {
moving++
} else {
still++
}
}
if cards >= 4 {
return QualityRejected, "多数画面是黑底文字广告", nil
}
if still >= 4 {
return QualityRejected, "多数片段为静止图片", nil
}
if moving >= 2 && cards <= 1 {
return QualityUsable, "检测到连续商品画面变化", nil
}
return QualityUnknown, "画面内容无法可靠判定", nil
}
func sampleFrame(ctx context.Context, ffmpeg, path string, at float64) (image.Image, error) {
cmd := exec.CommandContext(ctx, ffmpeg, "-hide_banner", "-loglevel", "error", "-ss", strconv.FormatFloat(at, 'f', 2, 64), "-i", path, "-frames:v", "1", "-vf", "scale=96:96", "-f", "image2pipe", "-vcodec", "png", "pipe:1")
cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: windows.CREATE_NO_WINDOW}
raw, err := cmd.Output()
if err != nil {
return nil, err
}
img, _, err := image.Decode(bytes.NewReader(raw))
return img, err
}
func textCard(img image.Image) bool {
b := img.Bounds()
total := b.Dx() * b.Dy()
if total == 0 {
return false
}
var dark, white, color int
for y := b.Min.Y; y < b.Max.Y; y++ {
for x := b.Min.X; x < b.Max.X; x++ {
r, g, bl, _ := img.At(x, y).RGBA()
rr, gg, bb := int(r>>8), int(g>>8), int(bl>>8)
if rr < 45 && gg < 45 && bb < 45 {
dark++
}
if rr > 180 && gg > 180 && bb > 180 {
white++
}
if max3(rr, gg, bb)-min3(rr, gg, bb) > 35 {
color++
}
}
}
return float64(dark)/float64(total) > .75 && float64(white)/float64(total) > .008 && float64(color)/float64(total) < .04
}
func frameDifference(a, b image.Image) float64 {
ra, rb := a.Bounds(), b.Bounds()
if ra.Dx() != rb.Dx() || ra.Dy() != rb.Dy() {
return 0
}
var sum float64
for y := 0; y < ra.Dy(); y++ {
for x := 0; x < ra.Dx(); x++ {
r1, g1, b1, _ := a.At(ra.Min.X+x, ra.Min.Y+y).RGBA()
r2, g2, b2, _ := b.At(rb.Min.X+x, rb.Min.Y+y).RGBA()
sum += math.Abs(float64(int(r1>>8)-int(r2>>8))) + math.Abs(float64(int(g1>>8)-int(g2>>8))) + math.Abs(float64(int(b1>>8)-int(b2>>8)))
}
}
return sum / float64(ra.Dx()*ra.Dy()*3)
}
func max3(a, b, c int) int {
if a < b {
a = b
}
if a < c {
a = c
}
return a
}
func min3(a, b, c int) int {
if a > b {
a = b
}
if a > c {
a = c
}
return a
}
+45
View File
@@ -0,0 +1,45 @@
package alibaba1688
import (
"image"
"image/color"
"testing"
)
func TestTextCardRejectsBlackBackgroundWithWhiteCopy(t *testing.T) {
img := image.NewRGBA(image.Rect(0, 0, 100, 100))
for y := 0; y < 100; y++ {
for x := 0; x < 100; x++ {
if x >= 20 && x < 80 && y >= 40 && y < 50 {
img.Set(x, y, color.RGBA{255, 255, 255, 255})
} else {
img.Set(x, y, color.RGBA{0, 0, 0, 255})
}
}
}
if !textCard(img) {
t.Fatal("black-background sales card should be detected")
}
for y := 0; y < 100; y++ {
for x := 0; x < 100; x++ {
img.Set(x, y, color.RGBA{60, 130, 100, 255})
}
}
if textCard(img) {
t.Fatal("colored product frame should not be a sales card")
}
}
func TestFrameDifferenceDetectsContinuousMotion(t *testing.T) {
a := image.NewRGBA(image.Rect(0, 0, 20, 20))
b := image.NewRGBA(image.Rect(0, 0, 20, 20))
for y := 0; y < 20; y++ {
for x := 0; x < 20; x++ {
a.Set(x, y, color.RGBA{20, 20, 20, 255})
b.Set(x, y, color.RGBA{100, 100, 100, 255})
}
}
if frameDifference(a, a) != 0 || frameDifference(a, b) < 6 {
t.Fatal("motion signal incorrect")
}
}
+292
View File
@@ -0,0 +1,292 @@
package alibaba1688
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"time"
"cmsp/internal/taobao"
)
var ErrLoginRequired = errors.New("1688 需要手动登录或完成安全验证")
var offerIDPattern = regexp.MustCompile(`^[0-9]{8,20}$`)
type target struct {
ID string `json:"id"`
URL string `json:"url"`
}
func targets(port int) ([]target, error) {
client := http.Client{Timeout: 3 * time.Second}
res, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/json/list", port))
if err != nil {
return nil, fmt.Errorf("读取 1688 Chrome 页面失败: %w", err)
}
defer res.Body.Close()
var values []target
if err := json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(&values); err != nil {
return nil, err
}
return values, nil
}
func fetchImage(ctx context.Context, raw string) (string, error) {
u, err := url.Parse(raw)
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
return "", fmt.Errorf("商品主图地址无效")
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
if err != nil {
return "", fmt.Errorf("商品主图地址无效")
}
req.Header.Set("User-Agent", "Mozilla/5.0")
client := http.Client{Timeout: 30 * time.Second}
res, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("下载商品主图失败")
}
defer res.Body.Close()
if res.StatusCode != 200 {
return "", fmt.Errorf("下载商品主图失败:HTTP %d", res.StatusCode)
}
f, err := os.CreateTemp("", "cmsp-1688-image-*.jpg")
if err != nil {
return "", err
}
defer f.Close()
n, err := io.Copy(f, io.LimitReader(res.Body, (10<<20)+1))
if err != nil || n > 10<<20 {
os.Remove(f.Name())
return "", fmt.Errorf("商品主图下载失败或超过 10 MB")
}
return f.Name(), nil
}
// SearchByImage uses the site's visible upload control, then reads public result cards.
// It never reads browser cookies or calls private search endpoints.
func SearchByImage(ctx context.Context, page *taobao.CDP, port int, imageURL string, limit int) ([]string, error) {
image, err := fetchImage(ctx, imageURL)
if err != nil {
return nil, err
}
defer os.Remove(image)
before, err := targets(port)
if err != nil {
return nil, err
}
seen := map[string]bool{}
for _, t := range before {
seen[t.ID] = true
}
if err := page.Navigate("https://www.1688.com/", 3*time.Second); err != nil {
return nil, err
}
if err := checkPage(page, false); err != nil {
return nil, err
}
var input struct {
NodeID int `json:"nodeId"`
}
for i := 0; i < 10 && input.NodeID == 0; i++ {
raw, err := page.Send("DOM.getDocument", map[string]any{"depth": 1})
if err != nil {
return nil, err
}
var doc struct {
Root struct {
NodeID int `json:"nodeId"`
} `json:"root"`
}
if err := json.Unmarshal(raw, &doc); err != nil {
return nil, err
}
raw, err = page.Send("DOM.querySelector", map[string]any{"nodeId": doc.Root.NodeID, "selector": `input[type="file"][accept*="jpg"]`})
if err != nil {
return nil, err
}
if err := json.Unmarshal(raw, &input); err != nil {
return nil, err
}
if input.NodeID == 0 {
if err := wait(ctx, time.Second); err != nil {
return nil, err
}
}
}
if input.NodeID == 0 {
return nil, fmt.Errorf("1688 以图搜上传框未找到")
}
if _, err := page.Send("DOM.setFileInputFiles", map[string]any{"nodeId": input.NodeID, "files": []string{image}}); err != nil {
return nil, err
}
var button string
for i := 0; i < 10; i++ {
if err := wait(ctx, time.Second); err != nil {
return nil, err
}
button, err = page.Evaluate(`(()=>{const b=[...document.querySelectorAll('button')].find(x=>x.textContent.trim()==='搜索'&&x.closest('[class*=imageSearch]'));if(!b||b.disabled||!b.closest('[class*=imageSearch]').innerText.includes('已上传1张图'))return '';const r=b.getBoundingClientRect();return JSON.stringify({x:r.x+r.width/2,y:r.y+r.height/2})})()`)
if err != nil {
return nil, err
}
if button != "" {
break
}
}
if button == "" {
return nil, fmt.Errorf("1688 以图搜图片未完成上传")
}
// The preview becomes visible before the site has finished preparing its search image.
if err := wait(ctx, 2*time.Second); err != nil {
return nil, err
}
var pos struct{ X, Y float64 }
if err := json.Unmarshal([]byte(button), &pos); err != nil {
return nil, err
}
if _, err := page.Send("Page.bringToFront", nil); err != nil {
return nil, err
}
for _, typ := range []string{"mouseMoved", "mousePressed", "mouseReleased"} {
if _, err := page.Send("Input.dispatchMouseEvent", map[string]any{"type": typ, "x": pos.X, "y": pos.Y, "button": "left", "clickCount": 1}); err != nil {
return nil, err
}
if typ == "mousePressed" {
if err := wait(ctx, 100*time.Millisecond); err != nil {
return nil, err
}
}
}
var resultURL, resultID string
for i := 0; i < 15 && resultURL == ""; i++ {
if err := wait(ctx, time.Second); err != nil {
return nil, err
}
list, err := targets(port)
if err != nil {
return nil, err
}
for _, t := range list {
if !seen[t.ID] && strings.HasPrefix(t.URL, "https://air.1688.com/kapp/1688-search/pc-image-search/") {
resultURL, resultID = t.URL, t.ID
break
}
}
}
if resultURL == "" {
return nil, fmt.Errorf("1688 图搜未打开结果页")
}
if err := page.Navigate(resultURL, 3*time.Second); err != nil {
return nil, err
}
if resultID != "" {
client := http.Client{Timeout: 2 * time.Second}
if response, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/json/close/%s", port, resultID)); err == nil {
response.Body.Close()
}
}
if err := checkPage(page, false); err != nil {
return nil, err
}
if limit < 10 {
limit = 10
}
expr := `JSON.stringify([...new Set([...document.querySelectorAll('[data-extra]')].map(x=>{try{return String(JSON.parse(x.dataset.extra).offerId||'')}catch(e){return ''}}).filter(Boolean))].slice(0,` + strconv.Itoa(limit) + `))`
var offers []string
for i := 0; i < 8; i++ {
text, err := page.Evaluate(expr)
if err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(text), &offers); err != nil {
return nil, err
}
if len(offers) > 0 {
break
}
if err := wait(ctx, time.Second); err != nil {
return nil, err
}
}
valid := offers[:0]
for _, id := range offers {
if offerIDPattern.MatchString(id) {
valid = append(valid, id)
}
}
return valid, nil
}
func OfferVideo(page *taobao.CDP, id string, delay time.Duration) (string, error) {
if !offerIDPattern.MatchString(id) {
return "", fmt.Errorf("1688 商品 ID 无效")
}
if err := page.Navigate("https://detail.1688.com/offer/"+id+".html", delay); err != nil {
return "", err
}
if err := checkPage(page, true); err != nil {
return "", err
}
text, err := page.Evaluate(`JSON.stringify([...document.querySelectorAll('video')].map(v=>v.currentSrc||v.src||'').filter(Boolean))`)
if err != nil {
return "", err
}
var values []string
if err := json.Unmarshal([]byte(text), &values); err != nil {
return "", err
}
for _, value := range values {
u, err := url.Parse(value)
if err != nil || u.Scheme != "https" || u.User != nil {
continue
}
host := strings.ToLower(u.Hostname())
if host == "cloud.video.taobao.com" || strings.HasSuffix(host, ".cloudvideocdn.com") {
return value, nil
}
}
return "", nil
}
func checkPage(page *taobao.CDP, detail bool) error {
raw, err := page.Evaluate(`JSON.stringify({url:location.href,title:document.title,text:(document.body?.innerText||'').slice(0,1200)})`)
if err != nil {
return err
}
var p struct{ URL, Title, Text string }
if err := json.Unmarshal([]byte(raw), &p); err != nil {
return err
}
u, err := url.Parse(p.URL)
if err != nil || !strings.HasSuffix(strings.ToLower(u.Hostname()), "1688.com") {
return ErrLoginRequired
}
if detail && (u.Hostname() != "detail.1688.com" || !strings.HasPrefix(u.Path, "/offer/")) {
return ErrLoginRequired
}
for _, word := range []string{"访问被拒绝", "安全验证", "请输入验证码", "请先登录"} {
if strings.Contains(p.Title, word) || strings.Contains(p.Text, word) {
return ErrLoginRequired
}
}
return nil
}
func wait(ctx context.Context, d time.Duration) error {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
return nil
}
}
+46 -5
View File
@@ -35,11 +35,12 @@ import (
// yaml tag 就是配置文件里的键名。改名会导致老配置读不出来,
// 所以除非有充分理由,不要改已有字段的 tag。
type Config struct {
ERPGo ERPGoConfig `yaml:"erpgo" json:"erpgo"`
Huohanhan HuohanhanConfig `yaml:"huohanhan" json:"huohanhan"`
Taobao TaobaoConfig `yaml:"taobao" json:"taobao"`
Download DownloadConfig `yaml:"download" json:"download"`
Upload UploadConfig `yaml:"upload" json:"upload"`
ERPGo ERPGoConfig `yaml:"erpgo" json:"erpgo"`
Huohanhan HuohanhanConfig `yaml:"huohanhan" json:"huohanhan"`
Taobao TaobaoConfig `yaml:"taobao" json:"taobao"`
Alibaba1688 TaobaoConfig `yaml:"alibaba1688" json:"alibaba1688"`
Download DownloadConfig `yaml:"download" json:"download"`
Upload UploadConfig `yaml:"upload" json:"upload"`
}
// ERPGoConfig 用于查询店铺和商品;APIKey 只保存在本机,禁止进入日志。
@@ -90,6 +91,7 @@ type TaobaoConfig struct {
// DownloadConfig 是下载与任务参数。
type DownloadConfig struct {
Source string `yaml:"source" json:"source"`
// VideoDir 是下载的视频保存目录。
VideoDir string `yaml:"video_dir" json:"videoDir"`
// MaxVideosPerProduct 是每个商品最多下载几个视频。
@@ -131,7 +133,14 @@ func Default() Config {
DebugPortStart: 19666,
DebugPortEnd: 19765,
},
Alibaba1688: TaobaoConfig{
ChromePath: `C:\Program Files\Google\Chrome\Application\chrome.exe`,
UserDataDir: filepath.Join(os.Getenv("LOCALAPPDATA"), "cmsp", "1688浏览器"),
DebugPortStart: 19766,
DebugPortEnd: 19865,
},
Download: DownloadConfig{
Source: "taobao",
VideoDir: DefaultVideoDir(),
MaxVideosPerProduct: 3,
SearchTopN: 5,
@@ -195,6 +204,24 @@ func (c Config) Validate() error {
}
// 旧 huohanhan 配置只为兼容现有 config.yaml 保留,不再用于请求。
t := c.Taobao
if c.Download.Source != "taobao" && c.Download.Source != "1688" {
return fmt.Errorf("下载来源须为 taobao 或 1688")
}
if strings.TrimSpace(c.Alibaba1688.ChromePath) == "" || strings.TrimSpace(c.Alibaba1688.UserDataDir) == "" {
return fmt.Errorf("1688 专属 Chrome 路径和用户数据目录不能为空")
}
if filepath.Clean(c.Alibaba1688.UserDataDir) == filepath.Clean(t.UserDataDir) {
return fmt.Errorf("1688 和淘宝不能共用 Chrome 用户数据目录")
}
if err := checkPort(c.Alibaba1688.DebugPortStart, "1688 调试端口起始"); err != nil {
return err
}
if err := checkPort(c.Alibaba1688.DebugPortEnd, "1688 调试端口结束"); err != nil {
return err
}
if c.Alibaba1688.DebugPortEnd < c.Alibaba1688.DebugPortStart {
return fmt.Errorf("1688 调试端口结束不能小于起始端口")
}
if strings.TrimSpace(t.ChromePath) == "" {
return fmt.Errorf("Chrome 可执行文件路径不能为空")
}
@@ -375,7 +402,16 @@ taobao:
debug_port_start: %d
debug_port_end: %d
alibaba1688:
# 1688 使用独立 Profile,登录只能在专属 Chrome 中手动完成。
chrome_path: %s
user_data_dir: %s
debug_port_start: %d
debug_port_end: %d
download:
# 下载来源:taobao 或 1688;仅影响新批次。
source: %s
# 下载的视频保存到哪个目录。
video_dir: %s
@@ -421,6 +457,11 @@ upload:
quoted(c.Taobao.UserDataDir),
c.Taobao.DebugPortStart,
c.Taobao.DebugPortEnd,
quoted(c.Alibaba1688.ChromePath),
quoted(c.Alibaba1688.UserDataDir),
c.Alibaba1688.DebugPortStart,
c.Alibaba1688.DebugPortEnd,
quoted(c.Download.Source),
quoted(c.Download.VideoDir),
c.Download.MaxVideosPerProduct,
c.Download.SearchTopN,
+20
View File
@@ -53,6 +53,26 @@ func TestDefaultConfigIsValid(t *testing.T) {
}
}
func TestVideoSourcePersistsAndKeepsIndependentBrowser(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.yaml")
cfg := Default()
cfg.Download.Source = "1688"
if cfg.Alibaba1688.UserDataDir == cfg.Taobao.UserDataDir {
t.Fatal("1688 and Taobao must not share Chrome profile")
}
if err := Save(path, cfg); err != nil {
t.Fatal(err)
}
loaded, err := Load(path)
if err != nil || loaded.Download.Source != "1688" || loaded.Alibaba1688.UserDataDir != cfg.Alibaba1688.UserDataDir {
t.Fatalf("source/browser config not persisted: %v", err)
}
cfg.Download.Source = "unknown"
if cfg.Validate() == nil {
t.Fatal("unsupported source accepted")
}
}
func TestValidateRejectsBadValues(t *testing.T) {
// 每个用例只改一个字段,这样失败时能一眼看出是哪条规则出问题。
cases := []struct {
+34 -3
View File
@@ -1,6 +1,7 @@
package store
import (
"database/sql"
"fmt"
"os"
"strings"
@@ -34,6 +35,39 @@ func (s *Store) ReplaceVideos(productID string, items []Video, now string) error
return fmt.Errorf("开启视频替换事务失败:%w", err)
}
defer tx.Rollback()
if err := replaceVideosTx(tx, productID, items, now); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("提交视频替换事务失败:%w", err)
}
return nil
}
// CompleteVideoDownload commits the accepted video and product status together.
func (s *Store) CompleteVideoDownload(productID string, video Video) error {
tx, err := s.db.Begin()
if err != nil {
return fmt.Errorf("开启视频下载完成事务失败:%w", err)
}
defer tx.Rollback()
if err := replaceVideosTx(tx, productID, []Video{video}, video.CreatedAt); err != nil {
return err
}
result, err := tx.Exec(`UPDATE products SET video_status = ?, download_status = ?, last_error = '' WHERE id = ?`, VideoFound, DownloadDone, productID)
if err != nil {
return fmt.Errorf("更新商品下载状态失败:%w", err)
}
if count, err := result.RowsAffected(); err != nil || count != 1 {
return fmt.Errorf("商品下载状态更新数量异常:%d,%v", count, err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("提交视频下载完成事务失败:%w", err)
}
return nil
}
func replaceVideosTx(tx *sql.Tx, productID string, items []Video, now string) error {
if _, err := tx.Exec(`DELETE FROM videos WHERE product_id = ?`, productID); err != nil {
return fmt.Errorf("清理商品旧视频失败:%w", err)
}
@@ -56,9 +90,6 @@ func (s *Store) ReplaceVideos(productID string, items []Video, now string) error
return fmt.Errorf("写入商品视频失败:%w", err)
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("提交视频替换事务失败:%w", err)
}
return nil
}
+25
View File
@@ -6,6 +6,31 @@ import (
"testing"
)
func TestCompleteVideoDownloadCommitsVideoAndStatusTogether(t *testing.T) {
s := newTestStore(t)
if _, err := s.DB().Exec(`INSERT INTO products (id, item_id, video_status, download_status) VALUES ('p1', 'fictional-1', 'pending', 'pending')`); err != nil {
t.Fatal(err)
}
video := Video{SourceItem: "fictional-1688-1", LocalPath: "D:/fictional.mp4", Status: VideoStatusDownloaded, CreatedAt: "2026-10-07 12:00:00"}
if err := s.CompleteVideoDownload("missing", video); err == nil {
t.Fatal("missing product must roll back")
}
if items, err := s.ListVideos("missing"); err != nil || len(items) != 0 {
t.Fatalf("rollback failed: %v %+v", err, items)
}
if err := s.CompleteVideoDownload("p1", video); err != nil {
t.Fatal(err)
}
product, found, err := s.GetProduct("p1")
if err != nil || !found || product.VideoStatus != VideoFound || product.DownloadStatus != DownloadDone {
t.Fatalf("status mismatch: %+v %v %v", product, found, err)
}
items, err := s.ListVideos("p1")
if err != nil || len(items) != 1 || items[0].SourceItem != video.SourceItem {
t.Fatalf("video mismatch: %+v %v", items, err)
}
}
func TestReplaceVideos按商品全量替换(t *testing.T) {
s := newTestStore(t)
first := []Video{
+9 -1
View File
@@ -35,6 +35,11 @@ type BrowserState struct {
// EnsureBrowser 复用归属明确且可连接的专属 Chrome;否则安全地启动一个新实例。
func EnsureBrowser(ctx context.Context, cfg config.TaobaoConfig, openURL string) (BrowserState, error) {
return EnsureBrowserWithState(ctx, cfg, openURL, browserStateName)
}
// EnsureBrowserWithState isolates runtime ownership metadata for another site/profile.
func EnsureBrowserWithState(ctx context.Context, cfg config.TaobaoConfig, openURL, stateName string) (BrowserState, error) {
if ctx == nil {
ctx = context.Background()
}
@@ -51,7 +56,10 @@ func EnsureBrowser(ctx context.Context, cfg config.TaobaoConfig, openURL string)
return BrowserState{}, fmt.Errorf("创建 Chrome 用户数据目录失败:%w", err)
}
stateFile := config.ResolveDataPath(browserStateName)
if stateName != browserStateName && stateName != "1688浏览器运行状态.json" {
return BrowserState{}, fmt.Errorf("不支持的浏览器状态文件")
}
stateFile := config.ResolveDataPath(stateName)
state, found, readErr := readBrowserState(stateFile)
if readErr == nil && found && browserStateReusable(state, profile) {
return state, nil
+8
View File
@@ -264,6 +264,14 @@ func (r *Runner) run(ctx context.Context, queue []queuedProduct) {
r.completeProduct(0, err == nil, err != nil)
} else {
downloaded, downloadErr := RunDownloads(ctx, r.options.Concurrency, work)
if errors.Is(downloadErr, ErrLoginRequired) || errors.Is(downloadErr, ErrRiskBlocked) {
r.setStopReason("login_required")
if errors.Is(downloadErr, ErrRiskBlocked) {
r.setStopReason(StopReasonRiskBlocked)
}
r.finish(StateLoginRequired)
return
}
r.completeProduct(downloaded, false, downloadErr != nil)
}
}