From 73f5688ab1296ecb84de9931793c86a5d59b2ce8 Mon Sep 17 00:00:00 2001 From: QiuSW <105186638@qq.com> Date: Thu, 3 Sep 2026 10:04:20 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=8D=95=E5=95=86=E5=93=81=E5=85=A8?= =?UTF-8?q?=E9=93=BE=E8=B7=AF=EF=BC=8C=E6=8F=90=E5=8F=96=E8=AF=A6=E6=83=85?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E5=B9=B6=E4=B8=8B=E8=BD=BD=E5=88=B0=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=20(#13)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 由 Codex (gpt-5.6-sol) 实施，Claude 审核。R4a，仍只处理单个商品。 internal/taobao/detail.go - 从同款详情页 HTML 正则提取 mp4 地址，还原 / 与 \/ 转义 - 只保留 cloud.video.taobao.com 与 cloudvideocdn 的地址 - 去重按（主机、路径、查询串）三元组，忽略 http/https 差异 internal/downloader - 先写 .part，ffprobe 校验通过后才改名为正式文件 - ffprobe 找不到时报明确中文错误，不静默跳过校验——跳过等于允许 损坏文件流入不可逆的上传环节 - 请求带 Referer（同款商品页）与浏览器 UA，否则淘宝 CDN 可能拒绝 - 目标文件已存在且非空时跳过 internal/store/video.go - 视频仓储，按商品全量替换 - source_item 记录实际采用的淘宝同款商品 ID（Q3 的落库要求） 同款选择按 Q3 结论：按图搜顺序依次尝试，第一个有视频的即采用， 不做人工确认；详情页之间按配置随机等待降低风控概率； 登录失效立即停止，不继续尝试下一个同款。 app.go 的 OpenFolder 用 EvalSymlinks + filepath.Rel 校验目标必须在 配置的 VideoDir 之内，避免成为可打开任意目录的入口。 审核补测（跑完删除，未入库）：下载中途断开、HTTP 403/404/500、 空响应三种情况均不得留下正式文件或 .part，实测全部通过； Referer 与 UA 确实发出。 审核期修正：Codex 又用 window.go.main.App 绕过生成绑定三处 （FetchVideosForProduct、OpenFolder、GetVideoSummary），已重新生成 绑定并改为正常 import。这是第三个工单出现同一问题。 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LbdtsD3ohhSMy3KPoCgARq --- app.go | 283 +++++++++++++++++++++++ docs/09-product-requirements-overview.md | 9 +- frontend/src/views/ProductListView.vue | 60 +++-- internal/downloader/downloader.go | 190 +++++++++++++++ internal/downloader/downloader_test.go | 101 ++++++++ internal/store/video.go | 83 +++++++ internal/store/video_test.go | 29 +++ internal/taobao/detail.go | 103 +++++++++ internal/taobao/detail_test.go | 47 ++++ 9 files changed, 887 insertions(+), 18 deletions(-) create mode 100644 internal/downloader/downloader.go create mode 100644 internal/downloader/downloader_test.go create mode 100644 internal/store/video.go create mode 100644 internal/store/video_test.go create mode 100644 internal/taobao/detail.go create mode 100644 internal/taobao/detail_test.go diff --git a/app.go b/app.go index 8a912be..b0f6bf9 100644 --- a/app.go +++ b/app.go @@ -2,11 +2,18 @@ package main import ( "context" + "errors" "fmt" + "math/rand" + "net/url" + "os" + "os/exec" + "path/filepath" "strings" "time" "cmsp/internal/config" + "cmsp/internal/downloader" "cmsp/internal/huohanhan" "cmsp/internal/logx" "cmsp/internal/store" @@ -336,6 +343,282 @@ func (a *App) SearchTaobaoByProduct(productID string) ([]taobao.SimilarItem, err return items, nil } +// FetchResult 是单个商品取视频的结果，供界面刷新数量和目录状态。 +type FetchResult struct { + ProductID string `json:"productId"` + SourceItem string `json:"sourceItem"` + VideoCount int `json:"videoCount"` + DownloadedCount int `json:"downloadedCount"` + Directory string `json:"directory"` +} + +// VideoSummary 是商品列表显示所需的最小本地视频摘要。 +type VideoSummary struct { + DownloadedCount int `json:"downloadedCount"` + Directory string `json:"directory"` +} + +// FetchVideosForProduct 为一个商品依次搜同款、提取视频并下载到本地。 +// 本方法始终串行执行，不创建批量任务、队列或并发下载。 +func (a *App) FetchVideosForProduct(productID string) (FetchResult, error) { + productID = strings.TrimSpace(productID) + result := FetchResult{ProductID: productID} + if a.db == nil { + return result, fmt.Errorf("数据库未就绪，请查看运行日志") + } + product, found, err := a.db.GetProduct(productID) + if err != nil { + return result, err + } + if !found { + return result, fmt.Errorf("找不到商品：%s", productID) + } + if strings.TrimSpace(product.MainImage) == "" { + return result, fmt.Errorf("商品没有可用于搜索的主图") + } + + ctx := a.ctx + if ctx == nil { + ctx = context.Background() + } + state, err := taobao.EnsureBrowser(ctx, a.cfg.Taobao, "https://www.taobao.com/") + if err != nil { + a.log.Error("商品 %s 准备淘宝专属 Chrome 失败：%v", product.ID, err) + return result, err + } + cdp, err := taobao.Connect(state.Port) + if err != nil { + a.log.Error("商品 %s 连接淘宝专属 Chrome 失败：%v", product.ID, err) + return result, err + } + defer cdp.Close() + + login, err := taobao.CheckLogin(cdp) + if err != nil { + login.Message = err.Error() + a.emitTaobaoStatus(login) + a.log.Error("商品 %s 取视频前深度检查淘宝登录失败：%v", product.ID, err) + return result, err + } + a.emitTaobaoStatus(login) + if !login.Valid { + err := fmt.Errorf("淘宝登录无效：%s", login.Message) + a.log.Warn("商品 %s 已触发登录停止门：%s", product.ID, login.Message) + return result, err + } + + // 登录通过后才清理旧记录，避免一次登录失效把已下载记录抹掉。 + if err := a.db.ReplaceVideos(product.ID, nil, time.Now().Format("2006-01-02 15:04:05")); err != nil { + return result, err + } + if err := a.db.UpdateProductStatus(product.ID, store.VideoPending, store.DownloadRunning, "", ""); err != nil { + return result, err + } + + a.log.Info("商品 %s 开始搜索淘宝同款", product.ID) + similar, err := taobao.SearchByImage(ctx, cdp, product.MainImage) + if err != nil { + return result, a.failVideoFetch(product.ID, fmt.Errorf("搜同款失败：%w", err)) + } + limit := min(len(similar), a.cfg.Download.SearchTopN) + a.log.Info("商品 %s 搜到 %d 个同款，将按顺序检查前 %d 个", product.ID, len(similar), limit) + + var selected taobao.SimilarItem + var videoURLs []string + var detailErrors []error + for i := 0; i < limit; i++ { + if i > 0 { + wait := randomWait(a.cfg.Download.WaitSecondsMin, a.cfg.Download.WaitSecondsMax) + a.log.Info("商品 %s 等待 %.1f 秒后检查下一个同款", product.ID, wait.Seconds()) + time.Sleep(wait) + } + item := similar[i] + a.log.Info("商品 %s 正在检查第 %d/%d 个淘宝同款 %s", product.ID, i+1, limit, item.ItemID) + urls, detailErr := taobao.ExtractDetailVideos(cdp, item.ItemID, 4*time.Second) + if errors.Is(detailErr, taobao.ErrLoginInvalid) { + a.emitTaobaoStatus(taobao.LoginStatus{Message: detailErr.Error()}) + a.log.Warn("商品 %s 访问详情页时登录失效，已立即停止", product.ID) + return result, a.failVideoFetch(product.ID, detailErr) + } + if detailErr != nil { + detailErrors = append(detailErrors, detailErr) + a.log.Warn("商品 %s 的同款 %s 详情读取失败：%v", product.ID, item.ItemID, detailErr) + continue + } + if len(urls) == 0 { + a.log.Info("淘宝同款 %s 未提取到可信视频", item.ItemID) + continue + } + selected = item + videoURLs = urls + break + } + + if len(videoURLs) == 0 { + if len(detailErrors) > 0 { + return result, a.failVideoFetch(product.ID, + fmt.Errorf("未能完成全部同款详情检查，首个错误：%w", detailErrors[0])) + } + if err := a.db.UpdateProductStatus(product.ID, store.VideoNone, store.DownloadPending, "", ""); err != nil { + return result, err + } + a.log.Info("商品 %s 检查完 %d 个同款，未找到视频", product.ID, limit) + return result, nil + } + + if len(videoURLs) > a.cfg.Download.MaxVideosPerProduct { + videoURLs = videoURLs[:a.cfg.Download.MaxVideosPerProduct] + } + videoDir, err := filepath.Abs(a.cfg.Download.VideoDir) + if err != nil { + return result, a.failVideoFetch(product.ID, fmt.Errorf("解析视频目录失败：%w", err)) + } + result.SourceItem = selected.ItemID + result.VideoCount = len(videoURLs) + result.Directory = videoDir + referer := "https://item.taobao.com/item.htm?id=" + url.QueryEscape(selected.ItemID) + downloaderClient := downloader.New() + records := make([]store.Video, 0, len(videoURLs)) + now := time.Now().Format("2006-01-02 15:04:05") + var firstDownloadError error + for i, sourceURL := range videoURLs { + name := downloader.Filename(product.ID, selected.ItemID, i+1) + target := filepath.Join(videoDir, name) + host := videoURLHost(sourceURL) + a.log.Info("商品 %s 正在下载视频 %d/%d（主机 %s，文件 %s）", + product.ID, i+1, len(videoURLs), host, name) + downloaded, downloadErr := downloaderClient.Download(ctx, sourceURL, referer, target) + record := store.Video{ + SourceItem: selected.ItemID, + SourceURL: sourceURL, + LocalPath: target, + Status: store.VideoStatusFailed, + CreatedAt: now, + } + if downloadErr != nil { + record.LastError = downloadErr.Error() + if firstDownloadError == nil { + firstDownloadError = downloadErr + } + a.log.Error("商品 %s 视频下载失败（主机 %s，文件 %s）：%v", + product.ID, host, name, downloadErr) + } else { + record.Status = store.VideoStatusDownloaded + record.FileSize = downloaded.Size + result.DownloadedCount++ + if downloaded.Skipped { + a.log.Success("商品 %s 视频已存在，跳过下载（文件 %s，%d 字节）", + product.ID, name, downloaded.Size) + } else { + a.log.Success("商品 %s 视频校验完成（文件 %s，时长 %.2f 秒，%d 字节）", + product.ID, name, downloaded.Duration, downloaded.Size) + } + } + records = append(records, record) + } + if err := a.db.ReplaceVideos(product.ID, records, now); err != nil { + return result, a.failVideoFetch(product.ID, err) + } + if firstDownloadError != nil { + if err := a.db.UpdateProductStatus(product.ID, store.VideoFound, store.DownloadFailed, "", firstDownloadError.Error()); err != nil { + return result, err + } + return result, firstDownloadError + } + if err := a.db.UpdateProductStatus(product.ID, store.VideoFound, store.DownloadDone, "", ""); err != nil { + return result, err + } + a.log.Success("商品 %s 采用同款 %s，共下载 %d 个视频", product.ID, selected.ItemID, result.DownloadedCount) + return result, nil +} + +func (a *App) failVideoFetch(productID string, cause error) error { + if err := a.db.UpdateProductStatus(productID, "", store.DownloadFailed, "", cause.Error()); err != nil { + a.log.Error("商品 %s 记录取视频失败状态时出错：%v", productID, err) + } + a.log.Error("商品 %s 取视频失败：%v", productID, cause) + return cause +} + +func randomWait(minSeconds, maxSeconds float64) time.Duration { + if maxSeconds <= minSeconds { + return time.Duration(minSeconds * float64(time.Second)) + } + seconds := minSeconds + rand.Float64()*(maxSeconds-minSeconds) + return time.Duration(seconds * float64(time.Second)) +} + +func videoURLHost(raw string) string { + parsed, err := url.Parse(raw) + if err != nil || parsed.Hostname() == "" { + return "未知" + } + return strings.ToLower(parsed.Hostname()) +} + +// GetVideoSummary 返回列表页需要的已下载视频数量与目录。 +func (a *App) GetVideoSummary(productID string) (VideoSummary, error) { + if a.db == nil { + return VideoSummary{}, fmt.Errorf("数据库未就绪，请查看运行日志") + } + items, err := a.db.ListVideos(strings.TrimSpace(productID)) + if err != nil { + return VideoSummary{}, err + } + summary := VideoSummary{} + for _, item := range items { + if item.Status != store.VideoStatusDownloaded || item.LocalPath == "" { + continue + } + info, statErr := os.Stat(item.LocalPath) + if statErr == nil && !info.IsDir() && info.Size() > 0 { + summary.DownloadedCount++ + } + } + if summary.DownloadedCount > 0 { + summary.Directory, err = filepath.Abs(a.cfg.Download.VideoDir) + if err != nil { + return VideoSummary{}, fmt.Errorf("解析视频目录失败：%w", err) + } + } + return summary, nil +} + +// OpenFolder 只允许打开配置的视频目录及其子目录。 +func (a *App) OpenFolder(path string) error { + base, err := filepath.Abs(a.cfg.Download.VideoDir) + if err != nil { + return fmt.Errorf("解析视频目录失败：%w", err) + } + target, err := filepath.Abs(strings.TrimSpace(path)) + if err != nil { + return fmt.Errorf("解析要打开的目录失败：%w", err) + } + baseResolved, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("视频目录不可访问：%w", err) + } + targetResolved, err := filepath.EvalSymlinks(target) + if err != nil { + return fmt.Errorf("目录不可访问：%w", err) + } + relative, err := filepath.Rel(baseResolved, targetResolved) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("只能打开视频保存目录及其子目录") + } + info, err := os.Stat(targetResolved) + if err != nil { + return fmt.Errorf("目录不可访问：%w", err) + } + if !info.IsDir() { + return fmt.Errorf("只能打开目录") + } + if err := exec.Command("explorer.exe", targetResolved).Start(); err != nil { + return fmt.Errorf("打开视频目录失败：%w", err) + } + return nil +} + // OpenInBrowser 用系统默认浏览器打开一个网址。 // // 只允许 http/https，避免以后有人把本地文件路径或自定义协议传进来 diff --git a/docs/09-product-requirements-overview.md b/docs/09-product-requirements-overview.md index a9c9e9b..b1742f6 100644 --- a/docs/09-product-requirements-overview.md +++ b/docs/09-product-requirements-overview.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Product-Requirements-Overview wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Product-Requirements-Overview.- -wiki_revision: a3a26399db6ba800b47955d0193867557e41081a -synchronized_at: 2026-09-02T08:07:24Z +wiki_revision: decc7537a5b852dbafa74bde6f51feaebedd9785 +synchronized_at: 2026-09-03T01:50:54Z # 需求总览 @@ -72,11 +72,13 @@ synchronized_at: 2026-09-02T08:07:24Z |---|---|---|---| | Q1 | 货憨憨素材上传的文件字段名，以及视频与图片是否走同一接口 | R5 | 待抓包确认 | | Q2 | `batchUpdateShopProductVideo` 的完整请求体字段与返回结构 | R5 | 只在货憨憨接口文档的接口清单中出现，无字段说明，待抓包确认 | -| Q3 | 一个商品搜到多个同款时，用哪一个的视频，是否需要人工确认 | R4、R6 | 待确认 | + | Q4 | 下载与上传的默认并发数，以及淘宝风控的实际容忍阈值 | R4、R7 | 待实测 | | Q5 | SQLite 表结构与迁移方式 | R8 | 待设计 | | Q6 | 上传后 Shopee 侧的生效延迟与验证方式 | R5 | 待实测 | +Q3 已于 2026-09-03 关闭：负责人实测图搜结果准确，决定批量全自动、不做人工确认，但采用的同款商品 ID 必须落库。 + Q1 与 Q2 是整条链路上唯一没有已验证参考实现的环节：仓库外的 Python 项目只实现了图片上传，视频上传接口没有代码；负责人提供的商品页 HAR 中这三个接口也出现 0 次。 **解决时机**：MVP2 交付后，使用者手动上传视频时抓包。在此之前 MVP3 不启动。 @@ -94,6 +96,7 @@ Q1 与 Q2 是整条链路上唯一没有已验证参考实现的环节：仓库 | 店铺来源 | 店铺下拉必须调用 `erp/shop/all` 获取当前账号店铺，不手工维护 | 账号可用店铺会变化 | | 运行日志 | 独立子窗口，主界面保留精简任务条（当前商品、进度、用时、停止按钮） | 任务运行时需同时看列表与进度 | | 路径选择 | Chrome 可执行文件用文件选择框（`*.exe`），用户数据目录与视频保存目录用目录选择框 | 手打 Windows 路径易错；两者是不同的对话框 API | +| 同款选择 | 批量时全自动：按图搜返回顺序依次尝试同款，第一个有视频的即采用，不做人工确认。但必须把实际采用的淘宝同款商品 ID 记入 `videos.source_item`，以便事后抽查与回溯 | 负责人 2026-09-03 确认图搜结果准确度可接受；几百个商品逐个人工确认不现实，记录来源即可满足回溯需要 | | 表格列 | 主图、蝦皮ID、标题、店铺、价格、视频、下载状态、上传状态、创建时间、操作 | 下载与上传状态分列，因为 `getPage` 不返回视频字段（见 Q7） | ## 登记规则 diff --git a/frontend/src/views/ProductListView.vue b/frontend/src/views/ProductListView.vue index 951ce47..555a821 100644 --- a/frontend/src/views/ProductListView.vue +++ b/frontend/src/views/ProductListView.vue @@ -27,13 +27,15 @@ import { NButton, useDialog, useMessage } from 'naive-ui' import { EventsOff, EventsOn } from '../../wailsjs/runtime/runtime' import { DownloadProductData, + FetchVideosForProduct, + GetVideoSummary, GetCachedShops, GetShopsUpdatedAt, RefreshShops, + OpenFolder, OpenInBrowser, DownloadVideos, ListProducts, - SearchTaobaoByProduct, UploadVideos, } from '../../wailsjs/go/main/App' @@ -200,6 +202,7 @@ const total = ref(0) const loading = ref(false) const checkedIds = ref([]) const searchingProductID = ref('') +const videoSummaries = ref({}) const similarModalVisible = ref(false) const similarItems = ref([]) const similarSourceTitle = ref('') @@ -235,7 +238,11 @@ const columns = [ title: '下载状态', key: 'downloadStatus', width: 110, - render: (row) => renderStatus(row.downloadStatus, downloadText), + render: (row) => { + const count = videoSummaries.value[row.id]?.downloadedCount || 0 + const status = renderStatus(row.downloadStatus, downloadText) + return h('span', [status, count > 0 ? ` · ${count}个` : '']) + }, }, { title: '上传状态', @@ -262,13 +269,13 @@ const columns = [ }, '详情' ), - // 目录：打开本地视频所在文件夹。视频还没下载时没有目录可开， - // 等 R4（工单 #5 MVP2）实现下载后再解开这里的禁用。 + // 目录：只有数据库记录对应的本地非空视频仍存在时才能打开。 h( 'a', { - class: 'op off', - title: '下载视频后可用（需求 R4）', + class: videoSummaries.value[row.id]?.directory ? 'op' : 'op off', + title: videoSummaries.value[row.id]?.directory || '尚无本地视频', + onClick: () => openVideoFolder(row), }, '目录' ), @@ -279,9 +286,9 @@ const columns = [ type: 'primary', loading: searchingProductID.value === row.id, disabled: searchingProductID.value !== '' && searchingProductID.value !== row.id, - onClick: () => searchSimilar(row), + onClick: () => fetchVideos(row), }, - { default: () => '搜同款' } + { default: () => '取视频' } ), ]) }, @@ -335,22 +342,44 @@ async function openShopee(row) { } } -async function searchSimilar(row) { +async function fetchVideos(row) { if (searchingProductID.value) return searchingProductID.value = row.id - similarSourceTitle.value = row.itemName || row.itemId || '当前商品' - similarItems.value = [] - similarError.value = '' - similarModalVisible.value = true + message.info('正在搜同款并下载视频，进度见运行日志') try { - similarItems.value = (await SearchTaobaoByProduct(row.id)) || [] + const result = await FetchVideosForProduct(row.id) + await search(false) + message.success(`取视频完成，下载了 ${result?.downloadedCount || 0} 个视频`) } catch (err) { - similarError.value = String(err) + message.error(`取视频失败：${err}`) } finally { searchingProductID.value = '' } } +async function openVideoFolder(row) { + const directory = videoSummaries.value[row.id]?.directory + if (!directory) return + try { + await OpenFolder(directory) + } catch (err) { + message.error(`打开视频目录失败：${err}`) + } +} + +async function loadVideoSummaries(items) { + const pairs = await Promise.all( + items.map(async (row) => { + try { + return [row.id, await GetVideoSummary(row.id)] + } catch (_) { + return [row.id, { downloadedCount: 0, directory: '' }] + } + }) + ) + videoSummaries.value = Object.fromEntries(pairs) +} + async function openTaobaoItem(item) { if (!item.url) return try { @@ -372,6 +401,7 @@ async function search(resetPage = true) { const page = await ListProducts(query.value) rows.value = page.items || [] total.value = page.total || 0 + await loadVideoSummaries(rows.value) } catch (err) { message.error(String(err)) } finally { diff --git a/internal/downloader/downloader.go b/internal/downloader/downloader.go new file mode 100644 index 0000000..ff47e83 --- /dev/null +++ b/internal/downloader/downloader.go @@ -0,0 +1,190 @@ +// Package downloader 提供单个视频的下载和 ffprobe 完整性校验。 +package downloader + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + "unicode" +) + +const downloadTimeout = 180 * time.Second + +type ProbeResult struct { + Duration float64 + Size int64 + FormatName string +} + +type ProbeFunc func(context.Context, string) (ProbeResult, error) + +type Result struct { + Path string + Size int64 + Duration float64 + Skipped bool +} + +type Downloader struct { + client *http.Client + probe ProbeFunc +} + +func New() *Downloader { + return &Downloader{ + client: &http.Client{Timeout: downloadTimeout}, + probe: runFFProbe, + } +} + +// NewWithOptions 只用于测试替换 HTTP 客户端和 ffprobe 执行。 +func NewWithOptions(client *http.Client, probe ProbeFunc) *Downloader { + if client == nil { + client = &http.Client{Timeout: downloadTimeout} + } + if probe == nil { + probe = runFFProbe + } + return &Downloader{client: client, probe: probe} +} + +// Filename 生成 Windows 可用且不会逃出目标目录的视频文件名。 +func Filename(productID, sourceItem string, index int) string { + if index < 1 { + index = 1 + } + return fmt.Sprintf("淘宝-%s-%s-%d.mp4", safeID(productID), safeID(sourceItem), index) +} + +func safeID(value string) string { + var b strings.Builder + for _, r := range strings.TrimSpace(value) { + if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' || r == '_' { + b.WriteRune(r) + } else { + b.WriteByte('_') + } + } + value = strings.Trim(b.String(), " .") + if value == "" { + return "unknown" + } + return value +} + +// Download 下载到 .part，ffprobe 校验成功后才改名为正式文件。 +func (d *Downloader) Download(ctx context.Context, sourceURL, referer, target string) (Result, error) { + if info, err := os.Stat(target); err == nil && !info.IsDir() && info.Size() > 0 { + return Result{Path: target, Size: info.Size(), Skipped: true}, nil + } + ctx, cancel := context.WithTimeout(ctx, downloadTimeout) + defer cancel() + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return Result{}, fmt.Errorf("创建视频目录失败：%w", err) + } + part := target + ".part" + _ = os.Remove(part) + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) + if err != nil { + return Result{}, fmt.Errorf("视频地址格式无效") + } + request.Header.Set("User-Agent", "Mozilla/5.0") + request.Header.Set("Referer", referer) + response, err := d.client.Do(request) + if err != nil { + // net/http 的错误通常包含完整 URL（包括查询签名），不能向日志上抛。 + return Result{}, fmt.Errorf("下载视频请求失败") + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + _, _ = io.Copy(io.Discard, response.Body) + return Result{}, fmt.Errorf("下载视频失败：HTTP %d", response.StatusCode) + } + + file, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return Result{}, fmt.Errorf("创建视频临时文件失败：%w", err) + } + _, copyErr := io.Copy(file, response.Body) + closeErr := file.Close() + if copyErr != nil { + _ = os.Remove(part) + return Result{}, fmt.Errorf("写入视频临时文件失败：%w", copyErr) + } + if closeErr != nil { + _ = os.Remove(part) + return Result{}, fmt.Errorf("关闭视频临时文件失败：%w", closeErr) + } + + probe, err := d.probe(ctx, part) + if err != nil { + _ = os.Remove(part) + return Result{}, fmt.Errorf("视频完整性校验失败：%w", err) + } + info, err := os.Stat(part) + if err != nil { + _ = os.Remove(part) + return Result{}, fmt.Errorf("读取视频临时文件失败：%w", err) + } + size := probe.Size + if size <= 0 { + size = info.Size() + } + if size <= 0 { + _ = os.Remove(part) + return Result{}, fmt.Errorf("视频完整性校验失败：文件大小为 0") + } + if err := os.Remove(target); err != nil && !os.IsNotExist(err) { + _ = os.Remove(part) + return Result{}, fmt.Errorf("替换空目标文件失败：%w", err) + } + if err := os.Rename(part, target); err != nil { + _ = os.Remove(part) + return Result{}, fmt.Errorf("保存正式视频文件失败：%w", err) + } + return Result{Path: target, Size: size, Duration: probe.Duration}, nil +} + +func runFFProbe(ctx context.Context, path string) (ProbeResult, error) { + ffprobe, err := exec.LookPath("ffprobe") + if err != nil { + return ProbeResult{}, fmt.Errorf("未找到 ffprobe，请先安装并加入 PATH") + } + command := exec.CommandContext(ctx, ffprobe, + "-v", "error", "-show_entries", "format=duration,size,format_name", "-of", "json", path) + output, err := command.Output() + if err != nil { + return ProbeResult{}, fmt.Errorf("ffprobe 执行失败：%w", err) + } + var payload struct { + Format struct { + Duration string `json:"duration"` + Size string `json:"size"` + FormatName string `json:"format_name"` + } `json:"format"` + } + if err := json.Unmarshal(output, &payload); err != nil { + return ProbeResult{}, fmt.Errorf("解析 ffprobe 输出失败：%w", err) + } + duration, err := strconv.ParseFloat(payload.Format.Duration, 64) + if err != nil || duration <= 0 { + return ProbeResult{}, fmt.Errorf("ffprobe 未返回有效 duration") + } + size, err := strconv.ParseInt(payload.Format.Size, 10, 64) + if err != nil || size <= 0 { + return ProbeResult{}, fmt.Errorf("ffprobe 未返回有效 size") + } + if strings.TrimSpace(payload.Format.FormatName) == "" { + return ProbeResult{}, fmt.Errorf("ffprobe 未返回 format_name") + } + return ProbeResult{Duration: duration, Size: size, FormatName: payload.Format.FormatName}, nil +} diff --git a/internal/downloader/downloader_test.go b/internal/downloader/downloader_test.go new file mode 100644 index 0000000..8f125f2 --- /dev/null +++ b/internal/downloader/downloader_test.go @@ -0,0 +1,101 @@ +package downloader + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" +) + +func Test文件名特殊字符不会生成非法路径(t *testing.T) { + got := Filename(`商品:/\\*?"<>|..`, `同款/A:B`, 2) + if strings.ContainsAny(got, `:/\\*?"<>|`) { + t.Fatalf("文件名仍包含 Windows 非法字符：%s", got) + } + if filepath.Base(got) != got || !strings.HasSuffix(got, ".mp4") { + t.Fatalf("文件名不应包含目录且必须以 mp4 结尾：%s", got) + } +} + +func Test已存在非空文件直接跳过下载(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = io.WriteString(w, "不应下载") + })) + defer server.Close() + target := filepath.Join(t.TempDir(), "已有.mp4") + if err := os.WriteFile(target, []byte("existing"), 0o644); err != nil { + t.Fatalf("准备已有文件失败：%v", err) + } + var probes atomic.Int32 + d := NewWithOptions(server.Client(), func(context.Context, string) (ProbeResult, error) { + probes.Add(1) + return ProbeResult{}, nil + }) + result, err := d.Download(context.Background(), server.URL, "https://item.taobao.com/", target) + if err != nil { + t.Fatalf("已有文件应当跳过：%v", err) + } + if !result.Skipped || requests.Load() != 0 || probes.Load() != 0 { + t.Fatalf("应跳过网络和校验：result=%+v requests=%d probes=%d", result, requests.Load(), probes.Load()) + } +} + +func Test校验失败只删除临时文件不留下正式文件(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("User-Agent") != "Mozilla/5.0" { + t.Errorf("User-Agent 不正确：%q", r.Header.Get("User-Agent")) + } + if r.Header.Get("Referer") != "https://item.taobao.com/item.htm?id=1" { + t.Errorf("Referer 不正确：%q", r.Header.Get("Referer")) + } + _, _ = io.WriteString(w, "broken video") + })) + defer server.Close() + target := filepath.Join(t.TempDir(), "失败.mp4") + d := NewWithOptions(server.Client(), func(context.Context, string) (ProbeResult, error) { + return ProbeResult{}, io.ErrUnexpectedEOF + }) + if _, err := d.Download(context.Background(), server.URL, "https://item.taobao.com/item.htm?id=1", target); err == nil { + t.Fatal("校验失败应当返回错误") + } + if _, err := os.Stat(target); !os.IsNotExist(err) { + t.Fatalf("校验失败不得留下正式文件：%v", err) + } + if _, err := os.Stat(target + ".part"); !os.IsNotExist(err) { + t.Fatalf("校验失败应删除 .part：%v", err) + } +} + +func Test下载校验成功后改名为正式文件(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "fake video bytes") + })) + defer server.Close() + target := filepath.Join(t.TempDir(), "成功.mp4") + d := NewWithOptions(server.Client(), func(_ context.Context, path string) (ProbeResult, error) { + if !strings.HasSuffix(path, ".part") { + t.Fatalf("ffprobe 必须校验 .part 文件：%s", path) + } + return ProbeResult{Duration: 12.5, Size: 16, FormatName: "mov,mp4"}, nil + }) + result, err := d.Download(context.Background(), server.URL, "https://item.taobao.com/", target) + if err != nil { + t.Fatalf("下载应成功：%v", err) + } + if result.Duration != 12.5 || result.Size != 16 { + t.Fatalf("校验结果未记录：%+v", result) + } + if info, err := os.Stat(target); err != nil || info.Size() == 0 { + t.Fatalf("正式文件不存在或为空：info=%v err=%v", info, err) + } + if _, err := os.Stat(target + ".part"); !os.IsNotExist(err) { + t.Fatalf("成功后不应留下 .part：%v", err) + } +} diff --git a/internal/store/video.go b/internal/store/video.go new file mode 100644 index 0000000..605e5b4 --- /dev/null +++ b/internal/store/video.go @@ -0,0 +1,83 @@ +package store + +import "fmt" + +const ( + VideoStatusPending = "pending" + VideoStatusDownloaded = "downloaded" + VideoStatusFailed = "failed" + VideoStatusUploaded = "uploaded" +) + +// Video 对应 videos 表的一行。 +type Video struct { + ID int64 `json:"id"` + ProductID string `json:"productId"` + SourceItem string `json:"sourceItem"` + SourceURL string `json:"sourceUrl"` + LocalPath string `json:"localPath"` + FileSize int64 `json:"fileSize"` + RemoteURL string `json:"remoteUrl"` + Status string `json:"status"` + LastError string `json:"lastError"` + CreatedAt string `json:"createdAt"` +} + +// ReplaceVideos 在一个事务内按商品全量替换视频记录。 +func (s *Store) ReplaceVideos(productID string, items []Video, now string) error { + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("开启视频替换事务失败：%w", err) + } + defer tx.Rollback() + if _, err := tx.Exec(`DELETE FROM videos WHERE product_id = ?`, productID); err != nil { + return fmt.Errorf("清理商品旧视频失败：%w", err) + } + stmt, err := tx.Prepare(`INSERT INTO videos ( + product_id, source_item, source_url, local_path, file_size, + remote_url, status, last_error, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("准备写入视频语句失败：%w", err) + } + defer stmt.Close() + for _, item := range items { + createdAt := item.CreatedAt + if createdAt == "" { + createdAt = now + } + if _, err := stmt.Exec(productID, item.SourceItem, item.SourceURL, + item.LocalPath, item.FileSize, item.RemoteURL, item.Status, + item.LastError, createdAt); err != nil { + return fmt.Errorf("写入商品视频失败：%w", err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("提交视频替换事务失败：%w", err) + } + return nil +} + +func (s *Store) ListVideos(productID string) ([]Video, error) { + rows, err := s.db.Query(`SELECT id, product_id, source_item, source_url, + local_path, file_size, remote_url, status, last_error, created_at + FROM videos WHERE product_id = ? ORDER BY id`, productID) + if err != nil { + return nil, fmt.Errorf("查询商品视频失败：%w", err) + } + defer rows.Close() + items := make([]Video, 0) + for rows.Next() { + var item Video + if err := rows.Scan(&item.ID, &item.ProductID, &item.SourceItem, + &item.SourceURL, &item.LocalPath, &item.FileSize, &item.RemoteURL, + &item.Status, &item.LastError, &item.CreatedAt); err != nil { + return nil, fmt.Errorf("读取商品视频行失败：%w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("遍历商品视频失败：%w", err) + } + return items, nil +} diff --git a/internal/store/video_test.go b/internal/store/video_test.go new file mode 100644 index 0000000..e4a9cf4 --- /dev/null +++ b/internal/store/video_test.go @@ -0,0 +1,29 @@ +package store + +import "testing" + +func TestReplaceVideos按商品全量替换(t *testing.T) { + s := newTestStore(t) + first := []Video{ + {SourceItem: "a", SourceURL: "https://example.invalid/a.mp4", Status: VideoStatusPending}, + {SourceItem: "b", SourceURL: "https://example.invalid/b.mp4", Status: VideoStatusDownloaded}, + {SourceItem: "c", SourceURL: "https://example.invalid/c.mp4", Status: VideoStatusFailed}, + } + if err := s.ReplaceVideos("product-1", first, "2026-09-03 10:00:00"); err != nil { + t.Fatalf("首次写入 3 条视频失败：%v", err) + } + second := []Video{{SourceItem: "d", SourceURL: "https://example.invalid/d.mp4", Status: VideoStatusDownloaded}} + if err := s.ReplaceVideos("product-1", second, "2026-09-03 11:00:00"); err != nil { + t.Fatalf("用 1 条视频替换失败：%v", err) + } + got, err := s.ListVideos("product-1") + if err != nil { + t.Fatalf("读取替换后视频失败：%v", err) + } + if len(got) != 1 || got[0].SourceItem != "d" { + t.Fatalf("全量替换后应只剩 d，实际：%+v", got) + } + if got[0].ProductID != "product-1" || got[0].CreatedAt != "2026-09-03 11:00:00" { + t.Fatalf("商品 ID 或创建时间没有按参数写入：%+v", got[0]) + } +} diff --git a/internal/taobao/detail.go b/internal/taobao/detail.go new file mode 100644 index 0000000..c11bea0 --- /dev/null +++ b/internal/taobao/detail.go @@ -0,0 +1,103 @@ +package taobao + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + "time" +) + +// ErrLoginInvalid 表示详情页访问过程中登录或安全验证状态失效。 +// 调用方遇到它必须立即停止，不能继续尝试下一个同款。 +var ErrLoginInvalid = errors.New("淘宝登录已失效") + +const detailVideoExpression = `(()=>{ + const h = document.documentElement.outerHTML; + return JSON.stringify({ + url: location.href, + title: document.title, + text: (document.body?.innerText || '').slice(0, 500), + urls: [...new Set( + (h.match(/https?:[^\"'<> ]+\.mp4[^\"'<> ]*/gi) || []) + .map(x => x.replaceAll('\\u002F','/').replaceAll('\\/','/')) + )] + }); +})()` + +type detailPage struct { + URL string `json:"url"` + Title string `json:"title"` + Text string `json:"text"` + URLs []string `json:"urls"` +} + +// ExtractDetailVideos 打开一个淘宝商品详情页并提取可信的视频 CDN 地址。 +func ExtractDetailVideos(cdp *CDP, itemID string, wait time.Duration) ([]string, error) { + itemID = strings.TrimSpace(itemID) + if itemID == "" { + return nil, fmt.Errorf("淘宝同款商品 ID 不能为空") + } + itemURL := "https://item.taobao.com/item.htm?id=" + url.QueryEscape(itemID) + if err := cdp.Navigate(itemURL, wait); err != nil { + return nil, fmt.Errorf("打开淘宝商品 %s 详情页失败：%w", itemID, err) + } + raw, err := cdp.Evaluate(detailVideoExpression) + if err != nil { + return nil, fmt.Errorf("读取淘宝商品 %s 详情页失败：%w", itemID, err) + } + var page detailPage + if err := json.Unmarshal([]byte(raw), &page); err != nil { + return nil, fmt.Errorf("淘宝商品 %s 详情页结果不是有效 JSON：%w", itemID, err) + } + if detailPageRequiresLogin(page) { + return nil, fmt.Errorf("%w：商品 %s 详情页要求重新登录或安全验证", ErrLoginInvalid, itemID) + } + return filterVideoURLs(page.URLs), nil +} + +func detailPageRequiresLogin(page detailPage) bool { + parsed, _ := url.Parse(page.URL) + host := strings.ToLower(parsed.Hostname()) + path := strings.ToLower(parsed.Path) + if strings.Contains(host, "login.taobao.com") || + strings.Contains(path, "/login") || strings.Contains(path, "/punish") { + return true + } + text := page.Title + "\n" + page.Text + for _, word := range loginBlockedWords { + if strings.Contains(text, word) { + return true + } + } + return false +} + +func filterVideoURLs(values []string) []string { + result := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + normalized := restoreJSONSlashes(strings.TrimSpace(value)) + parsed, err := url.Parse(normalized) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") { + continue + } + host := strings.ToLower(parsed.Hostname()) + if host != "cloud.video.taobao.com" && !strings.Contains(host, "cloudvideocdn") { + continue + } + key := host + "\x00" + parsed.EscapedPath() + "\x00" + parsed.RawQuery + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + result = append(result, normalized) + } + return result +} + +func restoreJSONSlashes(value string) string { + replacer := strings.NewReplacer(`\u002F`, "/", `\u002f`, "/", `\/`, "/") + return replacer.Replace(value) +} diff --git a/internal/taobao/detail_test.go b/internal/taobao/detail_test.go new file mode 100644 index 0000000..660f237 --- /dev/null +++ b/internal/taobao/detail_test.go @@ -0,0 +1,47 @@ +package taobao + +import ( + "reflect" + "testing" +) + +func Test视频地址过滤只保留淘宝视频CDN(t *testing.T) { + got := filterVideoURLs([]string{ + "https://cloud.video.taobao.com/play/a.mp4?token=1", + "https://cdn.example.com/ad.mp4", + "https://abc.cloudvideocdn.com/path/b.mp4", + "not-a-url.mp4", + }) + want := []string{ + "https://cloud.video.taobao.com/play/a.mp4?token=1", + "https://abc.cloudvideocdn.com/path/b.mp4", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("过滤结果不正确：got=%v want=%v", got, want) + } +} + +func Test视频地址去重忽略协议但保留不同查询串(t *testing.T) { + got := filterVideoURLs([]string{ + "http://cloud.video.taobao.com/a.mp4?k=1", + "https://CLOUD.VIDEO.TAOBAO.COM/a.mp4?k=1", + "https://cloud.video.taobao.com/a.mp4?k=2", + }) + want := []string{ + "http://cloud.video.taobao.com/a.mp4?k=1", + "https://cloud.video.taobao.com/a.mp4?k=2", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("去重结果不正确：got=%v want=%v", got, want) + } +} + +func Test视频地址还原JSON转义斜杠(t *testing.T) { + got := filterVideoURLs([]string{ + `https:\u002F\u002Fcloud.video.taobao.com\/path\/a.mp4?x=1`, + }) + want := []string{"https://cloud.video.taobao.com/path/a.mp4?x=1"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("JSON 转义斜杠未正确还原：got=%v want=%v", got, want) + } +}