From 6cc334ab929fe8e860cb9c095ae4641bdd8bbadc Mon Sep 17 00:00:00 2001 From: QiuSW <105186638@qq.com> Date: Thu, 3 Sep 2026 14:29:03 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=8F=96=E8=A7=86=E9=A2=91=E9=93=BE?= =?UTF-8?q?=E8=B7=AF=E4=B8=8E=20Python=20=E5=8F=82=E8=80=83=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=E5=AF=B9=E9=BD=90=EF=BC=8C=E8=AF=86=E5=88=AB=E9=A3=8E?= =?UTF-8?q?=E6=8E=A7=E9=99=8D=E7=BA=A7=20(#15)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 淘宝会在登录仍然有效的情况下停止下发视频资源：不跳登录页、不出验证码， 详情页里就是没有 mp4 地址。原实现识别不了，把它当成「这个同款没视频」， 继续跑满 20 个同款加重风控，最后写 video_status='none'， 而断点规则会永久跳过 none，导致账号恢复后这批商品也不再重试。 与参考实现对齐： - 详情页等待 4 秒改为 8 秒，提为 detail_wait_seconds - 每个同款详情页前做一次 my_itaobao 深度守卫，在两次详情页之间插入 正常页面访问；守卫等待提为 guard_wait_seconds - 详情页后补一次 Cookie 级快速守卫，复用 judgeLogin - 淘宝首页导航等待 3 秒改为 6 秒。签名请求在该页面上下文里 fetch， 页面没加载完就发可能带不上 Cookie - 下载失败重试，5xx 与连接错误重试 download_retries 次， 4xx 和 ffprobe 校验失败不重试 - 每次下载尝试各有独立的 180 秒超时。此前超时套在整个重试循环外面， 大视频首次跑到一半失败后剩余预算不足，重试等于不生效 新增风控降级识别：连续 risk_empty_threshold 个同款打开成功但取不到视频时， 判为疑似风控，中断整批任务并保持商品 pending，绝不写入 none。 停止原因通过 stopReason 枚举传给前端，不依赖中文文本分支。 数据订正提供显式按钮，不写进 migrations。刻意不做日期过滤： synced_at 记录的是商品数据何时从货憨憨拉取，与 video_status 何时被写成 none 无关，每次「下载数据」都会把它刷成当天。某商品是否需要重做的长期 答案来自货憨憨每次全量拉取覆盖的 video_diagnosis，不来自本地时间戳。 淘宝与 Chrome 相关逻辑由假实现覆盖，真机验证尚未进行。 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LbdtsD3ohhSMy3KPoCgARq --- app.go | 63 ++++++++++++++--- config.example.yaml | 12 ++++ frontend/src/views/ProductListView.vue | 7 +- frontend/src/views/SettingsView.vue | 94 +++++++++++++++++++++++++- internal/config/config.go | 40 ++++++++++- internal/config/config_test.go | 13 ++++ internal/downloader/downloader.go | 65 ++++++++++++++---- internal/downloader/downloader_test.go | 85 ++++++++++++++++++++++- internal/store/product.go | 32 +++++++++ internal/store/product_test.go | 56 +++++++++++++++ internal/taobao/detail.go | 22 ++++-- internal/taobao/imagesearch.go | 4 +- internal/taobao/login.go | 17 ++++- internal/task/task.go | 77 ++++++++++++++++++--- internal/task/task_test.go | 37 ++++++++++ 15 files changed, 570 insertions(+), 54 deletions(-) diff --git a/app.go b/app.go index 294cc3d..32565f3 100644 --- a/app.go +++ b/app.go @@ -206,7 +206,7 @@ func (a *App) CheckTaobaoLogin() (taobao.LoginStatus, error) { } defer cdp.Close() - status, err := taobao.CheckLogin(cdp) + status, err := taobao.CheckLogin(cdp, time.Duration(a.cfg.Download.GuardWaitSeconds*float64(time.Second))) if err != nil { status.Message = err.Error() a.emitTaobaoStatus(status) @@ -328,7 +328,7 @@ func (a *App) SearchTaobaoByProduct(productID string) ([]taobao.SimilarItem, err } defer cdp.Close() - status, err := taobao.CheckLogin(cdp) + status, err := taobao.CheckLogin(cdp, time.Duration(a.cfg.Download.GuardWaitSeconds*float64(time.Second))) if err != nil { status.Message = err.Error() a.emitTaobaoStatus(status) @@ -370,7 +370,8 @@ type VideoSummary struct { // 单商品入口与批量任务共用 prepareVideoFetch；这里保持原有串行下载行为。 func (a *App) FetchVideosForProduct(productID string) (FetchResult, error) { ctx := a.appContext() - result, work, err := a.prepareVideoFetch(ctx, productID) + riskGuard := task.NewEmptyRiskGuard(a.cfg.Download.RiskEmptyThreshold) + result, work, err := a.prepareVideoFetch(ctx, productID, riskGuard) if err != nil { return result, err } @@ -384,7 +385,7 @@ func (a *App) FetchVideosForProduct(productID string) (FetchResult, error) { // prepareVideoFetch 串行完成一个商品所有会触碰 CDP 的步骤，并把纯 HTTP // 下载闭包交给调用方。批量 Runner 因此不会让多个协程争抢同一个页面。 -func (a *App) prepareVideoFetch(ctx context.Context, productID string) (FetchResult, task.Work, error) { +func (a *App) prepareVideoFetch(ctx context.Context, productID string, riskGuard *task.EmptyRiskGuard) (FetchResult, task.Work, error) { productID = strings.TrimSpace(productID) result := FetchResult{ProductID: productID} if a.db == nil { @@ -413,7 +414,7 @@ func (a *App) prepareVideoFetch(ctx context.Context, productID string) (FetchRes } defer cdp.Close() - login, err := taobao.CheckLogin(cdp) + login, err := taobao.CheckLogin(cdp, time.Duration(a.cfg.Download.GuardWaitSeconds*float64(time.Second))) if err != nil { login.Message = err.Error() a.emitTaobaoStatus(login) @@ -453,7 +454,15 @@ func (a *App) prepareVideoFetch(ctx context.Context, productID string) (FetchRes } item := similar[i] a.log.Info("商品 %s 正在检查第 %d/%d 个淘宝同款 %s", product.ID, i+1, limit, item.ItemID) - urls, detailErr := taobao.ExtractDetailVideos(cdp, item.ItemID, 4*time.Second) + login, guardErr := taobao.CheckLogin(cdp, time.Duration(a.cfg.Download.GuardWaitSeconds*float64(time.Second))) + if guardErr != nil || !login.Valid { + message := login.Message + if guardErr != nil { + message = guardErr.Error() + } + return result, task.Work{}, fmt.Errorf("%w：%s", task.ErrLoginRequired, message) + } + detail, detailErr := taobao.ExtractDetailVideos(cdp, item.ItemID, time.Duration(a.cfg.Download.DetailWaitSeconds*float64(time.Second))) if errors.Is(detailErr, taobao.ErrLoginInvalid) { a.emitTaobaoStatus(taobao.LoginStatus{Message: detailErr.Error()}) a.log.Warn("商品 %s 访问详情页时登录失效，已立即停止", product.ID) @@ -464,12 +473,23 @@ func (a *App) prepareVideoFetch(ctx context.Context, productID string) (FetchRes a.log.Warn("商品 %s 的同款 %s 详情读取失败：%v", product.ID, item.ItemID, detailErr) continue } - if len(urls) == 0 { + quick, quickErr := taobao.QuickCheckLogin(cdp, detail.Title, detail.Text) + if quickErr != nil || !quick.Valid { + message := quick.Message + if quickErr != nil { + message = quickErr.Error() + } + return result, task.Work{}, fmt.Errorf("%w：%s", task.ErrLoginRequired, message) + } + if riskGuard != nil && riskGuard.Record(len(detail.URLs)) { + return result, task.Work{}, fmt.Errorf("%w：%w：%w", task.ErrLoginRequired, task.ErrRiskSuspected, task.RiskSuspectedError{Consecutive: riskGuard.Consecutive()}) + } + if len(detail.URLs) == 0 { a.log.Info("淘宝同款 %s 未提取到可信视频", item.ItemID) continue } selected = item - videoURLs = urls + videoURLs = detail.URLs break } @@ -515,7 +535,7 @@ func (a *App) prepareVideoFetch(ctx context.Context, productID string) (FetchRes host := videoURLHost(sourceURL) a.log.Info("商品 %s 正在下载视频 %d/%d（主机 %s，文件 %s）", product.ID, i+1, len(videoURLs), host, name) - downloaded, downloadErr := downloaderClient.Download(downloadCtx, sourceURL, referer, target) + downloaded, downloadErr := downloaderClient.Download(downloadCtx, sourceURL, referer, target, a.cfg.Download.DownloadRetries) record := store.Video{SourceItem: selected.ItemID, SourceURL: sourceURL, LocalPath: target, Status: store.VideoStatusFailed, CreatedAt: now} if downloadErr != nil { @@ -841,6 +861,7 @@ func (a *App) StartVideoTask(productIDs []string) error { return fmt.Errorf("已有任务在运行") } + riskGuard := task.NewEmptyRiskGuard(a.cfg.Download.RiskEmptyThreshold) runner := task.NewRunner(task.Options{ Concurrency: a.cfg.Download.Concurrency, WaitMin: time.Duration(a.cfg.Download.WaitSecondsMin * float64(time.Second)), @@ -862,7 +883,7 @@ func (a *App) StartVideoTask(productIDs []string) error { }, nil }, Prepare: func(ctx context.Context, product task.Product) (task.Work, error) { - _, work, err := a.prepareVideoFetch(ctx, product.ID) + _, work, err := a.prepareVideoFetch(ctx, product.ID, riskGuard) return work, err }, OnProgress: func(progress task.Progress) { @@ -885,6 +906,28 @@ func (a *App) StartVideoTask(productIDs []string) error { return runner.Start(a.appContext(), productIDs) } +// CountResettableNoneProducts 供设置页在人工确认前显示影响范围。 +func (a *App) CountResettableNoneProducts() (int, error) { + if a.db == nil { + return 0, fmt.Errorf("数据库未就绪，请查看运行日志") + } + return a.db.CountResettableNoneProducts() +} + +// ResetNoneProducts 仅在未运行取视频任务时执行显式数据订正。 +func (a *App) ResetNoneProducts() (int, error) { + if a.db == nil { + return 0, fmt.Errorf("数据库未就绪，请查看运行日志") + } + a.videoTaskMu.Lock() + runner := a.videoTask + a.videoTaskMu.Unlock() + if runner != nil && runner.Snapshot().State == task.StateRunning { + return 0, fmt.Errorf("取视频任务运行中，不能重置商品") + } + return a.db.ResetNoneProducts() +} + // StopVideoTask 请求在当前商品已启动的下载全部结束后停止。 func (a *App) StopVideoTask() error { a.videoTaskMu.Lock() diff --git a/config.example.yaml b/config.example.yaml index 60bf3c0..c43adb4 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -67,3 +67,15 @@ download: # 改小了跑得快但更容易被拦，不建议低于 2 秒。 wait_seconds_min: 2 wait_seconds_max: 4 + + # 淘宝详情页视频为异步加载；调小会漏视频，不建议低于 8 秒。 + detail_wait_seconds: 8 + + # 每个同款详情页前访问「我的淘宝」的深度登录守卫等待秒数。 + guard_wait_seconds: 3 + + # 连续多少个正常打开却没有视频的详情页时，判为疑似风控并停止任务。 + risk_empty_threshold: 8 + + # 网络层下载失败后的重试次数；HTTP 4xx 和 ffprobe 校验失败不会重试。 + download_retries: 3 diff --git a/frontend/src/views/ProductListView.vue b/frontend/src/views/ProductListView.vue index c2cc134..8b0cea2 100644 --- a/frontend/src/views/ProductListView.vue +++ b/frontend/src/views/ProductListView.vue @@ -599,9 +599,12 @@ async function onTaskLoginRequired(progress) { applyTaskProgress(progress) taskStopping.value = false await search(false) + const riskSuspected = progress?.stopReason === 'risk_suspected' dialog.warning({ - title: '淘宝登录已失效', - content: '淘宝登录已失效，整批任务已停止。\n已完成的部分不会重跑。请重新登录后再试', + title: riskSuspected ? '淘宝疑似风控' : '淘宝登录已失效', + content: riskSuspected + ? `淘宝疑似风控：登录仍然有效，但连续 ${progress?.riskEmptyCount || '多个'} 个同款都取不到视频。\n整批任务已停止，相关商品保持待处理，可稍后重试。` + : '淘宝登录已失效，整批任务已停止。\n已完成的部分不会重跑。请重新登录后再试', positiveText: '前往参数设置', negativeText: '稍后处理', onPositiveClick: goToSettings, diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index 82e347d..6894f4c 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -10,8 +10,9 @@ * 2. 路径不要求手打：chrome.exe 用文件选择框，两个目录用目录选择框。 * Windows 路径手打太容易错。 */ -import { onMounted, ref } from 'vue' -import { useMessage } from 'naive-ui' +import { onMounted, onUnmounted, ref } from 'vue' +import { useDialog, useMessage } from 'naive-ui' +import * as AppBindings from '../../wailsjs/go/main/App' import { CheckTaobaoLogin, GetConfig, @@ -23,11 +24,15 @@ import { } from '../../wailsjs/go/main/App' const message = useMessage() +const dialog = useDialog() const cfg = ref(null) const saving = ref(false) const openingTaobao = ref(false) const checkingTaobao = ref(false) const taobaoStatus = ref(null) +const resettingNoneProducts = ref(false) +const videoTaskRunning = ref(false) +let taskPollTimer = 0 async function load() { try { @@ -55,6 +60,43 @@ async function reset() { message.info('已还原默认值，点「保存设置」才会生效') } +async function resetNoneProducts() { + if (resettingNoneProducts.value) return + try { + const count = await AppBindings.CountResettableNoneProducts() + dialog.warning({ + title: '确认重置', + content: `将把 ${count} 条标记为「无同款视频」的商品退回待处理，` + + '它们会在下次批量任务中重新搜索淘宝。是否继续？', + positiveText: '继续重置', + negativeText: '取消', + onPositiveClick: async () => { + resettingNoneProducts.value = true + try { + const changed = await AppBindings.ResetNoneProducts() + message.success(`已重置 ${changed} 条商品`) + } catch (err) { + message.error(String(err)) + } finally { + resettingNoneProducts.value = false + } + }, + }) + } catch (err) { + message.error(String(err)) + } +} + +async function refreshTaskState() { + try { + const progress = await AppBindings.GetVideoTaskProgress() + videoTaskRunning.value = progress?.state === 'running' + } catch (_) { + // 无法读取状态时保守地不发起写操作；服务端也会再次拒绝运行中的请求。 + videoTaskRunning.value = true + } +} + // 使用者点了取消时后端返回空字符串，这时保持原值不动。 async function pickChrome() { const picked = await PickChromeExe() @@ -94,7 +136,14 @@ async function checkTaobaoLogin() { } } -onMounted(load) +onMounted(async () => { + await load() + await refreshTaskState() + taskPollTimer = window.setInterval(refreshTaskState, 1000) +}) +onUnmounted(() => { + if (taskPollTimer) window.clearInterval(taskPollTimer) +})