feat: 单商品全链路,提取详情视频并下载到本地 (#13)
由 Codex (gpt-5.6-sol) 实施,Claude 审核。R4a,仍只处理单个商品。 internal/taobao/detail.go - 从同款详情页 HTML 正则提取 mp4 地址,还原 / 与 \/ 转义 - 只保留 cloud.video.taobao.com 与 cloudvideocdn 的地址 - 去重按(主机、路径、查询串)三元组,忽略 http/https 差异 internal/downloader - 先写 .part,ffprobe 校验通过后才改名为正式文件 - ffprobe 找不到时报明确中文错误,不静默跳过校验——跳过等于允许 损坏文件流入不可逆的上传环节 - 请求带 Referer(同款商品页)与浏览器 UA,否则淘宝 CDN 可能拒绝 - 目标文件已存在且非空时跳过 internal/store/video.go - 视频仓储,按商品全量替换 - source_item 记录实际采用的淘宝同款商品 ID(Q3 的落库要求) 同款选择按 Q3 结论:按图搜顺序依次尝试,第一个有视频的即采用, 不做人工确认;详情页之间按配置随机等待降低风控概率; 登录失效立即停止,不继续尝试下一个同款。 app.go 的 OpenFolder 用 EvalSymlinks + filepath.Rel 校验目标必须在 配置的 VideoDir 之内,避免成为可打开任意目录的入口。 审核补测(跑完删除,未入库):下载中途断开、HTTP 403/404/500、 空响应三种情况均不得留下正式文件或 .part,实测全部通过; Referer 与 UA 确实发出。 审核期修正:Codex 又用 window.go.main.App 绕过生成绑定三处 (FetchVideosForProduct、OpenFolder、GetVideoSummary),已重新生成 绑定并改为正常 import。这是第三个工单出现同一问题。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LbdtsD3ohhSMy3KPoCgARq
This commit is contained in:
@@ -2,11 +2,18 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"cmsp/internal/config"
|
||||
"cmsp/internal/downloader"
|
||||
"cmsp/internal/huohanhan"
|
||||
"cmsp/internal/logx"
|
||||
"cmsp/internal/store"
|
||||
@@ -336,6 +343,282 @@ func (a *App) SearchTaobaoByProduct(productID string) ([]taobao.SimilarItem, err
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// FetchResult 是单个商品取视频的结果,供界面刷新数量和目录状态。
|
||||
type FetchResult struct {
|
||||
ProductID string `json:"productId"`
|
||||
SourceItem string `json:"sourceItem"`
|
||||
VideoCount int `json:"videoCount"`
|
||||
DownloadedCount int `json:"downloadedCount"`
|
||||
Directory string `json:"directory"`
|
||||
}
|
||||
|
||||
// VideoSummary 是商品列表显示所需的最小本地视频摘要。
|
||||
type VideoSummary struct {
|
||||
DownloadedCount int `json:"downloadedCount"`
|
||||
Directory string `json:"directory"`
|
||||
}
|
||||
|
||||
// FetchVideosForProduct 为一个商品依次搜同款、提取视频并下载到本地。
|
||||
// 本方法始终串行执行,不创建批量任务、队列或并发下载。
|
||||
func (a *App) FetchVideosForProduct(productID string) (FetchResult, error) {
|
||||
productID = strings.TrimSpace(productID)
|
||||
result := FetchResult{ProductID: productID}
|
||||
if a.db == nil {
|
||||
return result, fmt.Errorf("数据库未就绪,请查看运行日志")
|
||||
}
|
||||
product, found, err := a.db.GetProduct(productID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if !found {
|
||||
return result, fmt.Errorf("找不到商品:%s", productID)
|
||||
}
|
||||
if strings.TrimSpace(product.MainImage) == "" {
|
||||
return result, fmt.Errorf("商品没有可用于搜索的主图")
|
||||
}
|
||||
|
||||
ctx := a.ctx
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
state, err := taobao.EnsureBrowser(ctx, a.cfg.Taobao, "https://www.taobao.com/")
|
||||
if err != nil {
|
||||
a.log.Error("商品 %s 准备淘宝专属 Chrome 失败:%v", product.ID, err)
|
||||
return result, err
|
||||
}
|
||||
cdp, err := taobao.Connect(state.Port)
|
||||
if err != nil {
|
||||
a.log.Error("商品 %s 连接淘宝专属 Chrome 失败:%v", product.ID, err)
|
||||
return result, err
|
||||
}
|
||||
defer cdp.Close()
|
||||
|
||||
login, err := taobao.CheckLogin(cdp)
|
||||
if err != nil {
|
||||
login.Message = err.Error()
|
||||
a.emitTaobaoStatus(login)
|
||||
a.log.Error("商品 %s 取视频前深度检查淘宝登录失败:%v", product.ID, err)
|
||||
return result, err
|
||||
}
|
||||
a.emitTaobaoStatus(login)
|
||||
if !login.Valid {
|
||||
err := fmt.Errorf("淘宝登录无效:%s", login.Message)
|
||||
a.log.Warn("商品 %s 已触发登录停止门:%s", product.ID, login.Message)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// 登录通过后才清理旧记录,避免一次登录失效把已下载记录抹掉。
|
||||
if err := a.db.ReplaceVideos(product.ID, nil, time.Now().Format("2006-01-02 15:04:05")); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if err := a.db.UpdateProductStatus(product.ID, store.VideoPending, store.DownloadRunning, "", ""); err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
a.log.Info("商品 %s 开始搜索淘宝同款", product.ID)
|
||||
similar, err := taobao.SearchByImage(ctx, cdp, product.MainImage)
|
||||
if err != nil {
|
||||
return result, a.failVideoFetch(product.ID, fmt.Errorf("搜同款失败:%w", err))
|
||||
}
|
||||
limit := min(len(similar), a.cfg.Download.SearchTopN)
|
||||
a.log.Info("商品 %s 搜到 %d 个同款,将按顺序检查前 %d 个", product.ID, len(similar), limit)
|
||||
|
||||
var selected taobao.SimilarItem
|
||||
var videoURLs []string
|
||||
var detailErrors []error
|
||||
for i := 0; i < limit; i++ {
|
||||
if i > 0 {
|
||||
wait := randomWait(a.cfg.Download.WaitSecondsMin, a.cfg.Download.WaitSecondsMax)
|
||||
a.log.Info("商品 %s 等待 %.1f 秒后检查下一个同款", product.ID, wait.Seconds())
|
||||
time.Sleep(wait)
|
||||
}
|
||||
item := similar[i]
|
||||
a.log.Info("商品 %s 正在检查第 %d/%d 个淘宝同款 %s", product.ID, i+1, limit, item.ItemID)
|
||||
urls, detailErr := taobao.ExtractDetailVideos(cdp, item.ItemID, 4*time.Second)
|
||||
if errors.Is(detailErr, taobao.ErrLoginInvalid) {
|
||||
a.emitTaobaoStatus(taobao.LoginStatus{Message: detailErr.Error()})
|
||||
a.log.Warn("商品 %s 访问详情页时登录失效,已立即停止", product.ID)
|
||||
return result, a.failVideoFetch(product.ID, detailErr)
|
||||
}
|
||||
if detailErr != nil {
|
||||
detailErrors = append(detailErrors, detailErr)
|
||||
a.log.Warn("商品 %s 的同款 %s 详情读取失败:%v", product.ID, item.ItemID, detailErr)
|
||||
continue
|
||||
}
|
||||
if len(urls) == 0 {
|
||||
a.log.Info("淘宝同款 %s 未提取到可信视频", item.ItemID)
|
||||
continue
|
||||
}
|
||||
selected = item
|
||||
videoURLs = urls
|
||||
break
|
||||
}
|
||||
|
||||
if len(videoURLs) == 0 {
|
||||
if len(detailErrors) > 0 {
|
||||
return result, a.failVideoFetch(product.ID,
|
||||
fmt.Errorf("未能完成全部同款详情检查,首个错误:%w", detailErrors[0]))
|
||||
}
|
||||
if err := a.db.UpdateProductStatus(product.ID, store.VideoNone, store.DownloadPending, "", ""); err != nil {
|
||||
return result, err
|
||||
}
|
||||
a.log.Info("商品 %s 检查完 %d 个同款,未找到视频", product.ID, limit)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
if len(videoURLs) > a.cfg.Download.MaxVideosPerProduct {
|
||||
videoURLs = videoURLs[:a.cfg.Download.MaxVideosPerProduct]
|
||||
}
|
||||
videoDir, err := filepath.Abs(a.cfg.Download.VideoDir)
|
||||
if err != nil {
|
||||
return result, a.failVideoFetch(product.ID, fmt.Errorf("解析视频目录失败:%w", err))
|
||||
}
|
||||
result.SourceItem = selected.ItemID
|
||||
result.VideoCount = len(videoURLs)
|
||||
result.Directory = videoDir
|
||||
referer := "https://item.taobao.com/item.htm?id=" + url.QueryEscape(selected.ItemID)
|
||||
downloaderClient := downloader.New()
|
||||
records := make([]store.Video, 0, len(videoURLs))
|
||||
now := time.Now().Format("2006-01-02 15:04:05")
|
||||
var firstDownloadError error
|
||||
for i, sourceURL := range videoURLs {
|
||||
name := downloader.Filename(product.ID, selected.ItemID, i+1)
|
||||
target := filepath.Join(videoDir, name)
|
||||
host := videoURLHost(sourceURL)
|
||||
a.log.Info("商品 %s 正在下载视频 %d/%d(主机 %s,文件 %s)",
|
||||
product.ID, i+1, len(videoURLs), host, name)
|
||||
downloaded, downloadErr := downloaderClient.Download(ctx, sourceURL, referer, target)
|
||||
record := store.Video{
|
||||
SourceItem: selected.ItemID,
|
||||
SourceURL: sourceURL,
|
||||
LocalPath: target,
|
||||
Status: store.VideoStatusFailed,
|
||||
CreatedAt: now,
|
||||
}
|
||||
if downloadErr != nil {
|
||||
record.LastError = downloadErr.Error()
|
||||
if firstDownloadError == nil {
|
||||
firstDownloadError = downloadErr
|
||||
}
|
||||
a.log.Error("商品 %s 视频下载失败(主机 %s,文件 %s):%v",
|
||||
product.ID, host, name, downloadErr)
|
||||
} else {
|
||||
record.Status = store.VideoStatusDownloaded
|
||||
record.FileSize = downloaded.Size
|
||||
result.DownloadedCount++
|
||||
if downloaded.Skipped {
|
||||
a.log.Success("商品 %s 视频已存在,跳过下载(文件 %s,%d 字节)",
|
||||
product.ID, name, downloaded.Size)
|
||||
} else {
|
||||
a.log.Success("商品 %s 视频校验完成(文件 %s,时长 %.2f 秒,%d 字节)",
|
||||
product.ID, name, downloaded.Duration, downloaded.Size)
|
||||
}
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
if err := a.db.ReplaceVideos(product.ID, records, now); err != nil {
|
||||
return result, a.failVideoFetch(product.ID, err)
|
||||
}
|
||||
if firstDownloadError != nil {
|
||||
if err := a.db.UpdateProductStatus(product.ID, store.VideoFound, store.DownloadFailed, "", firstDownloadError.Error()); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, firstDownloadError
|
||||
}
|
||||
if err := a.db.UpdateProductStatus(product.ID, store.VideoFound, store.DownloadDone, "", ""); err != nil {
|
||||
return result, err
|
||||
}
|
||||
a.log.Success("商品 %s 采用同款 %s,共下载 %d 个视频", product.ID, selected.ItemID, result.DownloadedCount)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (a *App) failVideoFetch(productID string, cause error) error {
|
||||
if err := a.db.UpdateProductStatus(productID, "", store.DownloadFailed, "", cause.Error()); err != nil {
|
||||
a.log.Error("商品 %s 记录取视频失败状态时出错:%v", productID, err)
|
||||
}
|
||||
a.log.Error("商品 %s 取视频失败:%v", productID, cause)
|
||||
return cause
|
||||
}
|
||||
|
||||
func randomWait(minSeconds, maxSeconds float64) time.Duration {
|
||||
if maxSeconds <= minSeconds {
|
||||
return time.Duration(minSeconds * float64(time.Second))
|
||||
}
|
||||
seconds := minSeconds + rand.Float64()*(maxSeconds-minSeconds)
|
||||
return time.Duration(seconds * float64(time.Second))
|
||||
}
|
||||
|
||||
func videoURLHost(raw string) string {
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil || parsed.Hostname() == "" {
|
||||
return "未知"
|
||||
}
|
||||
return strings.ToLower(parsed.Hostname())
|
||||
}
|
||||
|
||||
// GetVideoSummary 返回列表页需要的已下载视频数量与目录。
|
||||
func (a *App) GetVideoSummary(productID string) (VideoSummary, error) {
|
||||
if a.db == nil {
|
||||
return VideoSummary{}, fmt.Errorf("数据库未就绪,请查看运行日志")
|
||||
}
|
||||
items, err := a.db.ListVideos(strings.TrimSpace(productID))
|
||||
if err != nil {
|
||||
return VideoSummary{}, err
|
||||
}
|
||||
summary := VideoSummary{}
|
||||
for _, item := range items {
|
||||
if item.Status != store.VideoStatusDownloaded || item.LocalPath == "" {
|
||||
continue
|
||||
}
|
||||
info, statErr := os.Stat(item.LocalPath)
|
||||
if statErr == nil && !info.IsDir() && info.Size() > 0 {
|
||||
summary.DownloadedCount++
|
||||
}
|
||||
}
|
||||
if summary.DownloadedCount > 0 {
|
||||
summary.Directory, err = filepath.Abs(a.cfg.Download.VideoDir)
|
||||
if err != nil {
|
||||
return VideoSummary{}, fmt.Errorf("解析视频目录失败:%w", err)
|
||||
}
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
// OpenFolder 只允许打开配置的视频目录及其子目录。
|
||||
func (a *App) OpenFolder(path string) error {
|
||||
base, err := filepath.Abs(a.cfg.Download.VideoDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析视频目录失败:%w", err)
|
||||
}
|
||||
target, err := filepath.Abs(strings.TrimSpace(path))
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析要打开的目录失败:%w", err)
|
||||
}
|
||||
baseResolved, err := filepath.EvalSymlinks(base)
|
||||
if err != nil {
|
||||
return fmt.Errorf("视频目录不可访问:%w", err)
|
||||
}
|
||||
targetResolved, err := filepath.EvalSymlinks(target)
|
||||
if err != nil {
|
||||
return fmt.Errorf("目录不可访问:%w", err)
|
||||
}
|
||||
relative, err := filepath.Rel(baseResolved, targetResolved)
|
||||
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return fmt.Errorf("只能打开视频保存目录及其子目录")
|
||||
}
|
||||
info, err := os.Stat(targetResolved)
|
||||
if err != nil {
|
||||
return fmt.Errorf("目录不可访问:%w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("只能打开目录")
|
||||
}
|
||||
if err := exec.Command("explorer.exe", targetResolved).Start(); err != nil {
|
||||
return fmt.Errorf("打开视频目录失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// OpenInBrowser 用系统默认浏览器打开一个网址。
|
||||
//
|
||||
// 只允许 http/https,避免以后有人把本地文件路径或自定义协议传进来
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
|
||||
wiki_page: Product-Requirements-Overview
|
||||
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Product-Requirements-Overview.-
|
||||
wiki_revision: a3a26399db6ba800b47955d0193867557e41081a
|
||||
synchronized_at: 2026-09-02T08:07:24Z
|
||||
wiki_revision: decc7537a5b852dbafa74bde6f51feaebedd9785
|
||||
synchronized_at: 2026-09-03T01:50:54Z
|
||||
<!-- gitea-wiki-mirror:end -->
|
||||
|
||||
# 需求总览
|
||||
@@ -72,11 +72,13 @@ synchronized_at: 2026-09-02T08:07:24Z
|
||||
|---|---|---|---|
|
||||
| Q1 | 货憨憨素材上传的文件字段名,以及视频与图片是否走同一接口 | R5 | 待抓包确认 |
|
||||
| Q2 | `batchUpdateShopProductVideo` 的完整请求体字段与返回结构 | R5 | 只在货憨憨接口文档的接口清单中出现,无字段说明,待抓包确认 |
|
||||
| Q3 | 一个商品搜到多个同款时,用哪一个的视频,是否需要人工确认 | R4、R6 | 待确认 |
|
||||
|
||||
| Q4 | 下载与上传的默认并发数,以及淘宝风控的实际容忍阈值 | R4、R7 | 待实测 |
|
||||
| Q5 | SQLite 表结构与迁移方式 | R8 | 待设计 |
|
||||
| Q6 | 上传后 Shopee 侧的生效延迟与验证方式 | R5 | 待实测 |
|
||||
|
||||
Q3 已于 2026-09-03 关闭:负责人实测图搜结果准确,决定批量全自动、不做人工确认,但采用的同款商品 ID 必须落库。
|
||||
|
||||
Q1 与 Q2 是整条链路上唯一没有已验证参考实现的环节:仓库外的 Python 项目只实现了图片上传,视频上传接口没有代码;负责人提供的商品页 HAR 中这三个接口也出现 0 次。
|
||||
|
||||
**解决时机**:MVP2 交付后,使用者手动上传视频时抓包。在此之前 MVP3 不启动。
|
||||
@@ -94,6 +96,7 @@ Q1 与 Q2 是整条链路上唯一没有已验证参考实现的环节:仓库
|
||||
| 店铺来源 | 店铺下拉必须调用 `erp/shop/all` 获取当前账号店铺,不手工维护 | 账号可用店铺会变化 |
|
||||
| 运行日志 | 独立子窗口,主界面保留精简任务条(当前商品、进度、用时、停止按钮) | 任务运行时需同时看列表与进度 |
|
||||
| 路径选择 | Chrome 可执行文件用文件选择框(`*.exe`),用户数据目录与视频保存目录用目录选择框 | 手打 Windows 路径易错;两者是不同的对话框 API |
|
||||
| 同款选择 | 批量时全自动:按图搜返回顺序依次尝试同款,第一个有视频的即采用,不做人工确认。但必须把实际采用的淘宝同款商品 ID 记入 `videos.source_item`,以便事后抽查与回溯 | 负责人 2026-09-03 确认图搜结果准确度可接受;几百个商品逐个人工确认不现实,记录来源即可满足回溯需要 |
|
||||
| 表格列 | 主图、蝦皮ID、标题、店铺、价格、视频、下载状态、上传状态、创建时间、操作 | 下载与上传状态分列,因为 `getPage` 不返回视频字段(见 Q7) |
|
||||
|
||||
## 登记规则
|
||||
|
||||
@@ -27,13 +27,15 @@ import { NButton, useDialog, useMessage } from 'naive-ui'
|
||||
import { EventsOff, EventsOn } from '../../wailsjs/runtime/runtime'
|
||||
import {
|
||||
DownloadProductData,
|
||||
FetchVideosForProduct,
|
||||
GetVideoSummary,
|
||||
GetCachedShops,
|
||||
GetShopsUpdatedAt,
|
||||
RefreshShops,
|
||||
OpenFolder,
|
||||
OpenInBrowser,
|
||||
DownloadVideos,
|
||||
ListProducts,
|
||||
SearchTaobaoByProduct,
|
||||
UploadVideos,
|
||||
} from '../../wailsjs/go/main/App'
|
||||
|
||||
@@ -200,6 +202,7 @@ const total = ref(0)
|
||||
const loading = ref(false)
|
||||
const checkedIds = ref([])
|
||||
const searchingProductID = ref('')
|
||||
const videoSummaries = ref({})
|
||||
const similarModalVisible = ref(false)
|
||||
const similarItems = ref([])
|
||||
const similarSourceTitle = ref('')
|
||||
@@ -235,7 +238,11 @@ const columns = [
|
||||
title: '下载状态',
|
||||
key: 'downloadStatus',
|
||||
width: 110,
|
||||
render: (row) => renderStatus(row.downloadStatus, downloadText),
|
||||
render: (row) => {
|
||||
const count = videoSummaries.value[row.id]?.downloadedCount || 0
|
||||
const status = renderStatus(row.downloadStatus, downloadText)
|
||||
return h('span', [status, count > 0 ? ` · ${count}个` : ''])
|
||||
},
|
||||
},
|
||||
{
|
||||
title: '上传状态',
|
||||
@@ -262,13 +269,13 @@ const columns = [
|
||||
},
|
||||
'详情'
|
||||
),
|
||||
// 目录:打开本地视频所在文件夹。视频还没下载时没有目录可开,
|
||||
// 等 R4(工单 #5 MVP2)实现下载后再解开这里的禁用。
|
||||
// 目录:只有数据库记录对应的本地非空视频仍存在时才能打开。
|
||||
h(
|
||||
'a',
|
||||
{
|
||||
class: 'op off',
|
||||
title: '下载视频后可用(需求 R4)',
|
||||
class: videoSummaries.value[row.id]?.directory ? 'op' : 'op off',
|
||||
title: videoSummaries.value[row.id]?.directory || '尚无本地视频',
|
||||
onClick: () => openVideoFolder(row),
|
||||
},
|
||||
'目录'
|
||||
),
|
||||
@@ -279,9 +286,9 @@ const columns = [
|
||||
type: 'primary',
|
||||
loading: searchingProductID.value === row.id,
|
||||
disabled: searchingProductID.value !== '' && searchingProductID.value !== row.id,
|
||||
onClick: () => searchSimilar(row),
|
||||
onClick: () => fetchVideos(row),
|
||||
},
|
||||
{ default: () => '搜同款' }
|
||||
{ default: () => '取视频' }
|
||||
),
|
||||
])
|
||||
},
|
||||
@@ -335,22 +342,44 @@ async function openShopee(row) {
|
||||
}
|
||||
}
|
||||
|
||||
async function searchSimilar(row) {
|
||||
async function fetchVideos(row) {
|
||||
if (searchingProductID.value) return
|
||||
searchingProductID.value = row.id
|
||||
similarSourceTitle.value = row.itemName || row.itemId || '当前商品'
|
||||
similarItems.value = []
|
||||
similarError.value = ''
|
||||
similarModalVisible.value = true
|
||||
message.info('正在搜同款并下载视频,进度见运行日志')
|
||||
try {
|
||||
similarItems.value = (await SearchTaobaoByProduct(row.id)) || []
|
||||
const result = await FetchVideosForProduct(row.id)
|
||||
await search(false)
|
||||
message.success(`取视频完成,下载了 ${result?.downloadedCount || 0} 个视频`)
|
||||
} catch (err) {
|
||||
similarError.value = String(err)
|
||||
message.error(`取视频失败:${err}`)
|
||||
} finally {
|
||||
searchingProductID.value = ''
|
||||
}
|
||||
}
|
||||
|
||||
async function openVideoFolder(row) {
|
||||
const directory = videoSummaries.value[row.id]?.directory
|
||||
if (!directory) return
|
||||
try {
|
||||
await OpenFolder(directory)
|
||||
} catch (err) {
|
||||
message.error(`打开视频目录失败:${err}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function loadVideoSummaries(items) {
|
||||
const pairs = await Promise.all(
|
||||
items.map(async (row) => {
|
||||
try {
|
||||
return [row.id, await GetVideoSummary(row.id)]
|
||||
} catch (_) {
|
||||
return [row.id, { downloadedCount: 0, directory: '' }]
|
||||
}
|
||||
})
|
||||
)
|
||||
videoSummaries.value = Object.fromEntries(pairs)
|
||||
}
|
||||
|
||||
async function openTaobaoItem(item) {
|
||||
if (!item.url) return
|
||||
try {
|
||||
@@ -372,6 +401,7 @@ async function search(resetPage = true) {
|
||||
const page = await ListProducts(query.value)
|
||||
rows.value = page.items || []
|
||||
total.value = page.total || 0
|
||||
await loadVideoSummaries(rows.value)
|
||||
} catch (err) {
|
||||
message.error(String(err))
|
||||
} finally {
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
// Package downloader 提供单个视频的下载和 ffprobe 完整性校验。
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
const downloadTimeout = 180 * time.Second
|
||||
|
||||
type ProbeResult struct {
|
||||
Duration float64
|
||||
Size int64
|
||||
FormatName string
|
||||
}
|
||||
|
||||
type ProbeFunc func(context.Context, string) (ProbeResult, error)
|
||||
|
||||
type Result struct {
|
||||
Path string
|
||||
Size int64
|
||||
Duration float64
|
||||
Skipped bool
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
client *http.Client
|
||||
probe ProbeFunc
|
||||
}
|
||||
|
||||
func New() *Downloader {
|
||||
return &Downloader{
|
||||
client: &http.Client{Timeout: downloadTimeout},
|
||||
probe: runFFProbe,
|
||||
}
|
||||
}
|
||||
|
||||
// NewWithOptions 只用于测试替换 HTTP 客户端和 ffprobe 执行。
|
||||
func NewWithOptions(client *http.Client, probe ProbeFunc) *Downloader {
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: downloadTimeout}
|
||||
}
|
||||
if probe == nil {
|
||||
probe = runFFProbe
|
||||
}
|
||||
return &Downloader{client: client, probe: probe}
|
||||
}
|
||||
|
||||
// Filename 生成 Windows 可用且不会逃出目标目录的视频文件名。
|
||||
func Filename(productID, sourceItem string, index int) string {
|
||||
if index < 1 {
|
||||
index = 1
|
||||
}
|
||||
return fmt.Sprintf("淘宝-%s-%s-%d.mp4", safeID(productID), safeID(sourceItem), index)
|
||||
}
|
||||
|
||||
func safeID(value string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range strings.TrimSpace(value) {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' || r == '_' {
|
||||
b.WriteRune(r)
|
||||
} else {
|
||||
b.WriteByte('_')
|
||||
}
|
||||
}
|
||||
value = strings.Trim(b.String(), " .")
|
||||
if value == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// Download 下载到 .part,ffprobe 校验成功后才改名为正式文件。
|
||||
func (d *Downloader) Download(ctx context.Context, sourceURL, referer, target string) (Result, error) {
|
||||
if info, err := os.Stat(target); err == nil && !info.IsDir() && info.Size() > 0 {
|
||||
return Result{Path: target, Size: info.Size(), Skipped: true}, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
|
||||
defer cancel()
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return Result{}, fmt.Errorf("创建视频目录失败:%w", err)
|
||||
}
|
||||
part := target + ".part"
|
||||
_ = os.Remove(part)
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("视频地址格式无效")
|
||||
}
|
||||
request.Header.Set("User-Agent", "Mozilla/5.0")
|
||||
request.Header.Set("Referer", referer)
|
||||
response, err := d.client.Do(request)
|
||||
if err != nil {
|
||||
// net/http 的错误通常包含完整 URL(包括查询签名),不能向日志上抛。
|
||||
return Result{}, fmt.Errorf("下载视频请求失败")
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
_, _ = io.Copy(io.Discard, response.Body)
|
||||
return Result{}, fmt.Errorf("下载视频失败:HTTP %d", response.StatusCode)
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("创建视频临时文件失败:%w", err)
|
||||
}
|
||||
_, copyErr := io.Copy(file, response.Body)
|
||||
closeErr := file.Close()
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(part)
|
||||
return Result{}, fmt.Errorf("写入视频临时文件失败:%w", copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(part)
|
||||
return Result{}, fmt.Errorf("关闭视频临时文件失败:%w", closeErr)
|
||||
}
|
||||
|
||||
probe, err := d.probe(ctx, part)
|
||||
if err != nil {
|
||||
_ = os.Remove(part)
|
||||
return Result{}, fmt.Errorf("视频完整性校验失败:%w", err)
|
||||
}
|
||||
info, err := os.Stat(part)
|
||||
if err != nil {
|
||||
_ = os.Remove(part)
|
||||
return Result{}, fmt.Errorf("读取视频临时文件失败:%w", err)
|
||||
}
|
||||
size := probe.Size
|
||||
if size <= 0 {
|
||||
size = info.Size()
|
||||
}
|
||||
if size <= 0 {
|
||||
_ = os.Remove(part)
|
||||
return Result{}, fmt.Errorf("视频完整性校验失败:文件大小为 0")
|
||||
}
|
||||
if err := os.Remove(target); err != nil && !os.IsNotExist(err) {
|
||||
_ = os.Remove(part)
|
||||
return Result{}, fmt.Errorf("替换空目标文件失败:%w", err)
|
||||
}
|
||||
if err := os.Rename(part, target); err != nil {
|
||||
_ = os.Remove(part)
|
||||
return Result{}, fmt.Errorf("保存正式视频文件失败:%w", err)
|
||||
}
|
||||
return Result{Path: target, Size: size, Duration: probe.Duration}, nil
|
||||
}
|
||||
|
||||
func runFFProbe(ctx context.Context, path string) (ProbeResult, error) {
|
||||
ffprobe, err := exec.LookPath("ffprobe")
|
||||
if err != nil {
|
||||
return ProbeResult{}, fmt.Errorf("未找到 ffprobe,请先安装并加入 PATH")
|
||||
}
|
||||
command := exec.CommandContext(ctx, ffprobe,
|
||||
"-v", "error", "-show_entries", "format=duration,size,format_name", "-of", "json", path)
|
||||
output, err := command.Output()
|
||||
if err != nil {
|
||||
return ProbeResult{}, fmt.Errorf("ffprobe 执行失败:%w", err)
|
||||
}
|
||||
var payload struct {
|
||||
Format struct {
|
||||
Duration string `json:"duration"`
|
||||
Size string `json:"size"`
|
||||
FormatName string `json:"format_name"`
|
||||
} `json:"format"`
|
||||
}
|
||||
if err := json.Unmarshal(output, &payload); err != nil {
|
||||
return ProbeResult{}, fmt.Errorf("解析 ffprobe 输出失败:%w", err)
|
||||
}
|
||||
duration, err := strconv.ParseFloat(payload.Format.Duration, 64)
|
||||
if err != nil || duration <= 0 {
|
||||
return ProbeResult{}, fmt.Errorf("ffprobe 未返回有效 duration")
|
||||
}
|
||||
size, err := strconv.ParseInt(payload.Format.Size, 10, 64)
|
||||
if err != nil || size <= 0 {
|
||||
return ProbeResult{}, fmt.Errorf("ffprobe 未返回有效 size")
|
||||
}
|
||||
if strings.TrimSpace(payload.Format.FormatName) == "" {
|
||||
return ProbeResult{}, fmt.Errorf("ffprobe 未返回 format_name")
|
||||
}
|
||||
return ProbeResult{Duration: duration, Size: size, FormatName: payload.Format.FormatName}, nil
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Test文件名特殊字符不会生成非法路径(t *testing.T) {
|
||||
got := Filename(`商品:/\\*?"<>|..`, `同款/A:B`, 2)
|
||||
if strings.ContainsAny(got, `:/\\*?"<>|`) {
|
||||
t.Fatalf("文件名仍包含 Windows 非法字符:%s", got)
|
||||
}
|
||||
if filepath.Base(got) != got || !strings.HasSuffix(got, ".mp4") {
|
||||
t.Fatalf("文件名不应包含目录且必须以 mp4 结尾:%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func Test已存在非空文件直接跳过下载(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
_, _ = io.WriteString(w, "不应下载")
|
||||
}))
|
||||
defer server.Close()
|
||||
target := filepath.Join(t.TempDir(), "已有.mp4")
|
||||
if err := os.WriteFile(target, []byte("existing"), 0o644); err != nil {
|
||||
t.Fatalf("准备已有文件失败:%v", err)
|
||||
}
|
||||
var probes atomic.Int32
|
||||
d := NewWithOptions(server.Client(), func(context.Context, string) (ProbeResult, error) {
|
||||
probes.Add(1)
|
||||
return ProbeResult{}, nil
|
||||
})
|
||||
result, err := d.Download(context.Background(), server.URL, "https://item.taobao.com/", target)
|
||||
if err != nil {
|
||||
t.Fatalf("已有文件应当跳过:%v", err)
|
||||
}
|
||||
if !result.Skipped || requests.Load() != 0 || probes.Load() != 0 {
|
||||
t.Fatalf("应跳过网络和校验:result=%+v requests=%d probes=%d", result, requests.Load(), probes.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func Test校验失败只删除临时文件不留下正式文件(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("User-Agent") != "Mozilla/5.0" {
|
||||
t.Errorf("User-Agent 不正确:%q", r.Header.Get("User-Agent"))
|
||||
}
|
||||
if r.Header.Get("Referer") != "https://item.taobao.com/item.htm?id=1" {
|
||||
t.Errorf("Referer 不正确:%q", r.Header.Get("Referer"))
|
||||
}
|
||||
_, _ = io.WriteString(w, "broken video")
|
||||
}))
|
||||
defer server.Close()
|
||||
target := filepath.Join(t.TempDir(), "失败.mp4")
|
||||
d := NewWithOptions(server.Client(), func(context.Context, string) (ProbeResult, error) {
|
||||
return ProbeResult{}, io.ErrUnexpectedEOF
|
||||
})
|
||||
if _, err := d.Download(context.Background(), server.URL, "https://item.taobao.com/item.htm?id=1", target); err == nil {
|
||||
t.Fatal("校验失败应当返回错误")
|
||||
}
|
||||
if _, err := os.Stat(target); !os.IsNotExist(err) {
|
||||
t.Fatalf("校验失败不得留下正式文件:%v", err)
|
||||
}
|
||||
if _, err := os.Stat(target + ".part"); !os.IsNotExist(err) {
|
||||
t.Fatalf("校验失败应删除 .part:%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func Test下载校验成功后改名为正式文件(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.WriteString(w, "fake video bytes")
|
||||
}))
|
||||
defer server.Close()
|
||||
target := filepath.Join(t.TempDir(), "成功.mp4")
|
||||
d := NewWithOptions(server.Client(), func(_ context.Context, path string) (ProbeResult, error) {
|
||||
if !strings.HasSuffix(path, ".part") {
|
||||
t.Fatalf("ffprobe 必须校验 .part 文件:%s", path)
|
||||
}
|
||||
return ProbeResult{Duration: 12.5, Size: 16, FormatName: "mov,mp4"}, nil
|
||||
})
|
||||
result, err := d.Download(context.Background(), server.URL, "https://item.taobao.com/", target)
|
||||
if err != nil {
|
||||
t.Fatalf("下载应成功:%v", err)
|
||||
}
|
||||
if result.Duration != 12.5 || result.Size != 16 {
|
||||
t.Fatalf("校验结果未记录:%+v", result)
|
||||
}
|
||||
if info, err := os.Stat(target); err != nil || info.Size() == 0 {
|
||||
t.Fatalf("正式文件不存在或为空:info=%v err=%v", info, err)
|
||||
}
|
||||
if _, err := os.Stat(target + ".part"); !os.IsNotExist(err) {
|
||||
t.Fatalf("成功后不应留下 .part:%v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package store
|
||||
|
||||
import "fmt"
|
||||
|
||||
const (
|
||||
VideoStatusPending = "pending"
|
||||
VideoStatusDownloaded = "downloaded"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUploaded = "uploaded"
|
||||
)
|
||||
|
||||
// Video 对应 videos 表的一行。
|
||||
type Video struct {
|
||||
ID int64 `json:"id"`
|
||||
ProductID string `json:"productId"`
|
||||
SourceItem string `json:"sourceItem"`
|
||||
SourceURL string `json:"sourceUrl"`
|
||||
LocalPath string `json:"localPath"`
|
||||
FileSize int64 `json:"fileSize"`
|
||||
RemoteURL string `json:"remoteUrl"`
|
||||
Status string `json:"status"`
|
||||
LastError string `json:"lastError"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
}
|
||||
|
||||
// ReplaceVideos 在一个事务内按商品全量替换视频记录。
|
||||
func (s *Store) ReplaceVideos(productID string, items []Video, now string) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("开启视频替换事务失败:%w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.Exec(`DELETE FROM videos WHERE product_id = ?`, productID); err != nil {
|
||||
return fmt.Errorf("清理商品旧视频失败:%w", err)
|
||||
}
|
||||
stmt, err := tx.Prepare(`INSERT INTO videos (
|
||||
product_id, source_item, source_url, local_path, file_size,
|
||||
remote_url, status, last_error, created_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("准备写入视频语句失败:%w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
for _, item := range items {
|
||||
createdAt := item.CreatedAt
|
||||
if createdAt == "" {
|
||||
createdAt = now
|
||||
}
|
||||
if _, err := stmt.Exec(productID, item.SourceItem, item.SourceURL,
|
||||
item.LocalPath, item.FileSize, item.RemoteURL, item.Status,
|
||||
item.LastError, createdAt); err != nil {
|
||||
return fmt.Errorf("写入商品视频失败:%w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("提交视频替换事务失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ListVideos(productID string) ([]Video, error) {
|
||||
rows, err := s.db.Query(`SELECT id, product_id, source_item, source_url,
|
||||
local_path, file_size, remote_url, status, last_error, created_at
|
||||
FROM videos WHERE product_id = ? ORDER BY id`, productID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询商品视频失败:%w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]Video, 0)
|
||||
for rows.Next() {
|
||||
var item Video
|
||||
if err := rows.Scan(&item.ID, &item.ProductID, &item.SourceItem,
|
||||
&item.SourceURL, &item.LocalPath, &item.FileSize, &item.RemoteURL,
|
||||
&item.Status, &item.LastError, &item.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("读取商品视频行失败:%w", err)
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("遍历商品视频失败:%w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestReplaceVideos按商品全量替换(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
first := []Video{
|
||||
{SourceItem: "a", SourceURL: "https://example.invalid/a.mp4", Status: VideoStatusPending},
|
||||
{SourceItem: "b", SourceURL: "https://example.invalid/b.mp4", Status: VideoStatusDownloaded},
|
||||
{SourceItem: "c", SourceURL: "https://example.invalid/c.mp4", Status: VideoStatusFailed},
|
||||
}
|
||||
if err := s.ReplaceVideos("product-1", first, "2026-09-03 10:00:00"); err != nil {
|
||||
t.Fatalf("首次写入 3 条视频失败:%v", err)
|
||||
}
|
||||
second := []Video{{SourceItem: "d", SourceURL: "https://example.invalid/d.mp4", Status: VideoStatusDownloaded}}
|
||||
if err := s.ReplaceVideos("product-1", second, "2026-09-03 11:00:00"); err != nil {
|
||||
t.Fatalf("用 1 条视频替换失败:%v", err)
|
||||
}
|
||||
got, err := s.ListVideos("product-1")
|
||||
if err != nil {
|
||||
t.Fatalf("读取替换后视频失败:%v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].SourceItem != "d" {
|
||||
t.Fatalf("全量替换后应只剩 d,实际:%+v", got)
|
||||
}
|
||||
if got[0].ProductID != "product-1" || got[0].CreatedAt != "2026-09-03 11:00:00" {
|
||||
t.Fatalf("商品 ID 或创建时间没有按参数写入:%+v", got[0])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package taobao
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrLoginInvalid 表示详情页访问过程中登录或安全验证状态失效。
|
||||
// 调用方遇到它必须立即停止,不能继续尝试下一个同款。
|
||||
var ErrLoginInvalid = errors.New("淘宝登录已失效")
|
||||
|
||||
const detailVideoExpression = `(()=>{
|
||||
const h = document.documentElement.outerHTML;
|
||||
return JSON.stringify({
|
||||
url: location.href,
|
||||
title: document.title,
|
||||
text: (document.body?.innerText || '').slice(0, 500),
|
||||
urls: [...new Set(
|
||||
(h.match(/https?:[^\"'<> ]+\.mp4[^\"'<> ]*/gi) || [])
|
||||
.map(x => x.replaceAll('\\u002F','/').replaceAll('\\/','/'))
|
||||
)]
|
||||
});
|
||||
})()`
|
||||
|
||||
type detailPage struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Text string `json:"text"`
|
||||
URLs []string `json:"urls"`
|
||||
}
|
||||
|
||||
// ExtractDetailVideos 打开一个淘宝商品详情页并提取可信的视频 CDN 地址。
|
||||
func ExtractDetailVideos(cdp *CDP, itemID string, wait time.Duration) ([]string, error) {
|
||||
itemID = strings.TrimSpace(itemID)
|
||||
if itemID == "" {
|
||||
return nil, fmt.Errorf("淘宝同款商品 ID 不能为空")
|
||||
}
|
||||
itemURL := "https://item.taobao.com/item.htm?id=" + url.QueryEscape(itemID)
|
||||
if err := cdp.Navigate(itemURL, wait); err != nil {
|
||||
return nil, fmt.Errorf("打开淘宝商品 %s 详情页失败:%w", itemID, err)
|
||||
}
|
||||
raw, err := cdp.Evaluate(detailVideoExpression)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取淘宝商品 %s 详情页失败:%w", itemID, err)
|
||||
}
|
||||
var page detailPage
|
||||
if err := json.Unmarshal([]byte(raw), &page); err != nil {
|
||||
return nil, fmt.Errorf("淘宝商品 %s 详情页结果不是有效 JSON:%w", itemID, err)
|
||||
}
|
||||
if detailPageRequiresLogin(page) {
|
||||
return nil, fmt.Errorf("%w:商品 %s 详情页要求重新登录或安全验证", ErrLoginInvalid, itemID)
|
||||
}
|
||||
return filterVideoURLs(page.URLs), nil
|
||||
}
|
||||
|
||||
func detailPageRequiresLogin(page detailPage) bool {
|
||||
parsed, _ := url.Parse(page.URL)
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
path := strings.ToLower(parsed.Path)
|
||||
if strings.Contains(host, "login.taobao.com") ||
|
||||
strings.Contains(path, "/login") || strings.Contains(path, "/punish") {
|
||||
return true
|
||||
}
|
||||
text := page.Title + "\n" + page.Text
|
||||
for _, word := range loginBlockedWords {
|
||||
if strings.Contains(text, word) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func filterVideoURLs(values []string) []string {
|
||||
result := make([]string, 0, len(values))
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
normalized := restoreJSONSlashes(strings.TrimSpace(value))
|
||||
parsed, err := url.Parse(normalized)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||||
continue
|
||||
}
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
if host != "cloud.video.taobao.com" && !strings.Contains(host, "cloudvideocdn") {
|
||||
continue
|
||||
}
|
||||
key := host + "\x00" + parsed.EscapedPath() + "\x00" + parsed.RawQuery
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
result = append(result, normalized)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func restoreJSONSlashes(value string) string {
|
||||
replacer := strings.NewReplacer(`\u002F`, "/", `\u002f`, "/", `\/`, "/")
|
||||
return replacer.Replace(value)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package taobao
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Test视频地址过滤只保留淘宝视频CDN(t *testing.T) {
|
||||
got := filterVideoURLs([]string{
|
||||
"https://cloud.video.taobao.com/play/a.mp4?token=1",
|
||||
"https://cdn.example.com/ad.mp4",
|
||||
"https://abc.cloudvideocdn.com/path/b.mp4",
|
||||
"not-a-url.mp4",
|
||||
})
|
||||
want := []string{
|
||||
"https://cloud.video.taobao.com/play/a.mp4?token=1",
|
||||
"https://abc.cloudvideocdn.com/path/b.mp4",
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("过滤结果不正确:got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func Test视频地址去重忽略协议但保留不同查询串(t *testing.T) {
|
||||
got := filterVideoURLs([]string{
|
||||
"http://cloud.video.taobao.com/a.mp4?k=1",
|
||||
"https://CLOUD.VIDEO.TAOBAO.COM/a.mp4?k=1",
|
||||
"https://cloud.video.taobao.com/a.mp4?k=2",
|
||||
})
|
||||
want := []string{
|
||||
"http://cloud.video.taobao.com/a.mp4?k=1",
|
||||
"https://cloud.video.taobao.com/a.mp4?k=2",
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("去重结果不正确:got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func Test视频地址还原JSON转义斜杠(t *testing.T) {
|
||||
got := filterVideoURLs([]string{
|
||||
`https:\u002F\u002Fcloud.video.taobao.com\/path\/a.mp4?x=1`,
|
||||
})
|
||||
want := []string{"https://cloud.video.taobao.com/path/a.mp4?x=1"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("JSON 转义斜杠未正确还原:got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user