From b8453acfc6eff4f24cb9d1242301798ebdd2f2dd Mon Sep 17 00:00:00 2001 From: QiuSW <105186638@qq.com> Date: Tue, 29 Sep 2026 15:39:32 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=9D=E7=95=99=E5=9B=BE=E6=90=9C?= =?UTF-8?q?=E5=8E=9F=E5=A7=8B=E5=95=86=E5=93=81=E9=93=BE=E6=8E=A5=E7=94=A8?= =?UTF-8?q?=E4=BA=8E=E8=AF=A6=E6=83=85=E5=AF=BC=E8=88=AA=20(#23)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app.go | 2 +- app_video_test.go | 2 +- docs/02-architecture-and-code-map.md | 10 ++- docs/03-business-rules-and-glossary.md | 10 ++- docs/04-local-development-and-verification.md | 10 ++- docs/06-troubleshooting.md | 10 ++- internal/taobao/access_test.go | 2 +- internal/taobao/candidates.go | 10 ++- internal/taobao/detail.go | 38 +++++++++-- internal/taobao/detail_test.go | 65 +++++++++++++++++++ internal/taobao/imagesearch.go | 22 +++++-- internal/taobao/imagesearch_test.go | 20 ++++++ 12 files changed, 178 insertions(+), 23 deletions(-) diff --git a/app.go b/app.go index 5a23238..2562926 100644 --- a/app.go +++ b/app.go @@ -542,7 +542,7 @@ func (a *App) prepareVideoFetchOnPage(ctx context.Context, product store.Product result.SourceItem = selected.ItemID result.VideoCount = len(videoURLs) result.Directory = productDir - referer := "https://item.taobao.com/item.htm?id=" + url.QueryEscape(selected.ItemID) + referer := selected.URL downloaderClient := downloader.New() records := make([]store.Video, len(videoURLs)) now := time.Now().Format("2006-01-02 15:04:05") diff --git a/app_video_test.go b/app_video_test.go index 1881517..2b1da5c 100644 --- a/app_video_test.go +++ b/app_video_test.go @@ -89,7 +89,7 @@ func TestRiskStopPreservesSQLiteProductVideosAndRemainingQueue(t *testing.T) { if stage == "search" { return nil, taobao.ErrAccessBlocked } - return []taobao.SimilarItem{{ItemID: "1"}, {ItemID: "2"}}, nil + return []taobao.SimilarItem{{ItemID: "1", URL: "https://detail.tmall.com/item.htm?ns=1&id=1&skuId=fictional"}, {ItemID: "2"}}, nil } finished := make(chan task.Progress, 1) guard := task.NewEmptyRiskGuard(1) diff --git a/docs/02-architecture-and-code-map.md b/docs/02-architecture-and-code-map.md index 2a8a073..25fdeee 100644 --- a/docs/02-architecture-and-code-map.md +++ b/docs/02-architecture-and-code-map.md @@ -2,12 +2,18 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Architecture-and-Code-Map wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Architecture-and-Code-Map.- -wiki_revision: e9367a04bc7af1ffbf956c9ba406fdd99f0df8ee -synchronized_at: 2026-09-29T03:33:19Z +wiki_revision: 662a06a06a45623d4adb68c5bf1fe86ee70103ee +synchronized_at: 2026-09-29T07:39:06Z # 架构与代码地图 +## 图搜原始商品链接（2026-09-29） + +`parseImageSearchResult` 读取已通过真实响应确认的 `itemsArray[].auctionURL`，写入 `SimilarItem.URL`。`SimilarItem.DetailURL` 在 Go 侧校验 http/https、精确商品域名 `item.taobao.com` / `detail.tmall.com`、`/item.htm` 路径及唯一且匹配的 `id`，拒绝用户信息、其他端口、片段、错误编码和不可信站点；协议相对地址补 https。保留原链接查询参数与顺序，不记录完整链接或参数值。只有响应缺少链接时兼容原 ID 拼接方式；非法原链接候选跳过，不访问也不改拼旧地址，其他合法候选照常返回；若有非法链接且全部候选都不可用，则明确报错，不能记为正常无结果。 + +`SelectVideo` 校验链接后传整个 `SimilarItem` 给 `ExtractDetailVideos`，详情导航使用原链接；选中的 `SimilarItem.URL` 同时用于 CDN 下载 Referer。原有串行候选、同批 ID 缓存、登录/访问异常停止门和 SQLite 状态保持不变。这里仍是浏览器直接导航到返回的商品链接：MTOP 搜索没有生成淘宝搜索结果页，也没有模拟结果卡片点击或为详情导航额外设置 Referer。 + ## 淘宝访问异常停止与候选复用（2026-09-29） `App.prepareVideoFetch` 建立专属 Chrome 连接；`prepareVideoFetchOnPage` 调用本批 `taobao.LoginGuard.Check`：首次实际处理调用 `CheckLogin` 打开「我的淘宝」深度检查，后续商品调用 `CheckCurrentLogin` 读取当前 URL/标题/有限正文和 Cookie 名称，不发送检查导航。图搜首页、MTOP 返回和详情页中的明确访问异常均映射到任务全局停止门，签名算法及请求参数不变。 diff --git a/docs/03-business-rules-and-glossary.md b/docs/03-business-rules-and-glossary.md index 1455a91..934f6f9 100644 --- a/docs/03-business-rules-and-glossary.md +++ b/docs/03-business-rules-and-glossary.md @@ -2,12 +2,18 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Business-Rules-and-Glossary wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Business-Rules-and-Glossary.- -wiki_revision: 6fa4581023a61ca0d5479cf46d696688b4d266f3 -synchronized_at: 2026-09-29T03:33:20Z +wiki_revision: f4dcb37885891ef200f0d4703d97d623fa8aac47 +synchronized_at: 2026-09-29T07:39:09Z # 业务规则与术语 +## 同款商品链接使用（2026-09-29） + +MTOP 搜索方式、签名与请求参数保持现状。商品原链接以实际返回字段 `auctionURL` 为准，保留查询参数；允许淘宝/天猫的商品详情 `/item.htm`，必须与候选商品 ID 一致。非法链接候选跳过，不导航、不改拼旧地址、不泄漏到错误日志；其他合法候选仍可处理，全为非法链接则明确报错，不能记为正常无结果。缺少字段时才使用原 ID 拼接链接。完整链接仅用于当前查询结果、详情导航与下载 Referer，不新增持久化任务状态。 + +使用返回链接直接导航不等同于从淘宝结果页点击。当前未获得可直接打开的结果页地址证据，不猜测构造结果页/来源参数，不自动收藏商品，不承诺解除访问异常。访问异常、登录失效仍即时停批，不自动重试或恢复轮询。 + ## 淘宝访问与停止规则（2026-09-29） - 明确访问异常：页面出现「当前访问存在异常」「恢复正常访问方式」「访问受限」、安全验证/验证码或 `/punish` 路径时立即停批，`stopReason=risk_blocked`。即使 Cookie 齐全或页面残留视频 URL，也不继续访问。 diff --git a/docs/04-local-development-and-verification.md b/docs/04-local-development-and-verification.md index 7f33435..33d7f77 100644 --- a/docs/04-local-development-and-verification.md +++ b/docs/04-local-development-and-verification.md @@ -2,12 +2,18 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Local-Development-and-Verification wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Local-Development-and-Verification.- -wiki_revision: 5027339b211f8d7762449b4e579a4886a2e8bff4 -synchronized_at: 2026-09-29T03:33:20Z +wiki_revision: a99cdc64887622e4127aebd0693d37fffaa3f0c4 +synchronized_at: 2026-09-29T07:39:11Z # 本地开发与验证 +## 商品原链接验证（2026-09-29） + +`internal/taobao/imagesearch_test.go` 验证 `auctionURL` 参数保留、缺失字段兼容、非法链接候选跳过及全部非法时明确报错；`detail_test.go` 验证淘宝/天猫域名、商品 ID 匹配、拒绝恶意域名/重复 ID/错误编码/用户信息/其他端口及片段、原链接导航、缓存复用和受限后停止。`app_video_test.go` 的 SQLite 停止场景包含天猫原链接，验证商品、视频、上传及剩余队列状态保留。常规测试不访问淘宝。 + +从根目录执行 `go test ./...`、`go test -race ./...`、`go vet ./...`；Windows 构建可用 `& "$env:USERPROFILE\go\bin\wails.exe" build -o cmsp23.exe`。不要与正在更新前端嵌入资源的构建同时运行 Go 测试。真实联调须先确认专属浏览器归属、登录检查通过且没有进行中的下载任务；只读 SQLite，响应只分析必要字段名/链接结构，不落盘原始响应与凭据。遇到请求或访问异常即停止，不为验证重复请求。 + ## 淘宝风控停止门验证（2026-09-29） 从仓库根目录先完成前端资源构建，再执行嵌入这些资源的 Go 测试，避免两者同时更新/读取 `frontend/dist`： diff --git a/docs/06-troubleshooting.md b/docs/06-troubleshooting.md index a71a11d..4fa2952 100644 --- a/docs/06-troubleshooting.md +++ b/docs/06-troubleshooting.md @@ -2,12 +2,18 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Troubleshooting wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Troubleshooting -wiki_revision: 8e56ccb600bd56584ca513fe87bd2cd05f4328da -synchronized_at: 2026-09-29T03:33:21Z +wiki_revision: c91a79baab19786e491b46e8aceb2308a2ea49d5 +synchronized_at: 2026-09-29T07:39:14Z # 故障排查 +## 图搜成功但商品链接无法使用（2026-09-29） + +程序优先使用 MTOP 返回的 `auctionURL`，不会丢弃原查询参数后重新按 ID 拼接。单个非法链接候选会跳过，不影响其他合法候选；如果所有候选都不可用，报“淘宝以图搜返回了无效商品链接”，检查经过脱敏的字段结构是否发生变化；只允许淘宝/天猫商品域名、`/item.htm` 和与候选匹配的唯一 `id`。不能关闭校验、盲信广告跳转域名或把完整响应/参数写入日志。缺少链接字段的旧响应仍兼容按 ID 打开。 + +保留原链接是进入路径调整，不代表浏览器已经处于淘宝搜索结果页，也不能证明平台风控缓解。受限提示仍按下方停止门处理。仅确认商品链接成功打开不足以证明视频属于当前商品或下载完成。 + ## 淘宝提示「当前访问存在异常」（2026-09-29） 先停止自动任务。程序检测到此提示或安全验证时应显示「淘宝访问受限」，整批停止并保留断点；不会将受限详情记为无视频，也不会自动重试。不要连续点击启动。 diff --git a/internal/taobao/access_test.go b/internal/taobao/access_test.go index 735caa2..f4c7962 100644 --- a/internal/taobao/access_test.go +++ b/internal/taobao/access_test.go @@ -126,7 +126,7 @@ func TestAccessPromptStopsDespiteCookiesAndVideo(t *testing.T) { } for _, url := range []string{"https://item.taobao.com/punish?fictional=secret", "https://login.taobao.com/member/login.jhtml"} { p := &fakePage{pages: map[string]detailPage{"https://item.taobao.com/item.htm?id=1": {URL: url}}} - _, err := ExtractDetailVideos(p, "1", 0) + _, err := ExtractDetailVideos(p, SimilarItem{ItemID: "1"}, 0) want := ErrLoginInvalid if strings.Contains(url, "punish") { want = ErrAccessBlocked diff --git a/internal/taobao/candidates.go b/internal/taobao/candidates.go index b286fe7..76e6af9 100644 --- a/internal/taobao/candidates.go +++ b/internal/taobao/candidates.go @@ -35,6 +35,14 @@ func SelectVideo(ctx context.Context, page PageSession, items []SimilarItem, opt } seen[item.ItemID] = true checked++ + itemURL, err := item.DetailURL() + if err != nil { + if firstError == nil { + firstError = err + } + continue + } + item.URL = itemURL urls, cached := cache[item.ItemID] if !cached { if visited { @@ -50,7 +58,7 @@ func SelectVideo(ctx context.Context, page PageSession, items []SimilarItem, opt return SimilarItem{}, nil, err } visited = true - detail, err := ExtractDetailVideos(page, item.ItemID, options.DetailWait) + detail, err := ExtractDetailVideos(page, item, options.DetailWait) if canceled := ctx.Err(); canceled != nil { return SimilarItem{}, nil, canceled } diff --git a/internal/taobao/detail.go b/internal/taobao/detail.go index 0423d2d..d60b499 100644 --- a/internal/taobao/detail.go +++ b/internal/taobao/detail.go @@ -42,12 +42,12 @@ type DetailResult struct { } // ExtractDetailVideos 打开一个淘宝商品详情页并提取可信的视频 CDN 地址。 -func ExtractDetailVideos(cdp PageSession, itemID string, wait time.Duration) (DetailResult, error) { - itemID = strings.TrimSpace(itemID) - if itemID == "" { - return DetailResult{}, fmt.Errorf("淘宝同款商品 ID 不能为空") +func ExtractDetailVideos(cdp PageSession, item SimilarItem, wait time.Duration) (DetailResult, error) { + itemURL, err := item.DetailURL() + if err != nil { + return DetailResult{}, err } - itemURL := "https://item.taobao.com/item.htm?id=" + url.QueryEscape(itemID) + itemID := item.ItemID if err := cdp.Navigate(itemURL, wait); err != nil { return DetailResult{}, fmt.Errorf("打开淘宝商品 %s 详情页失败：%w", itemID, err) } @@ -68,6 +68,34 @@ func ExtractDetailVideos(cdp PageSession, itemID string, wait time.Duration) (De return DetailResult{Title: page.Title, Text: page.Text, URLs: filterVideoURLs(page.URLs)}, nil } +// DetailURL 保留实际图搜返回的链接参数；仅旧响应缺少链接时按 ID 兼容。 +// 校验发生在 Go 侧，不能让外部链接把专属登录会话引向其他站点或商品。 +func (item SimilarItem) DetailURL() (string, error) { + itemID := strings.TrimSpace(item.ItemID) + if itemID == "" { + return "", fmt.Errorf("淘宝同款商品 ID 不能为空") + } + raw := strings.TrimSpace(item.URL) + if raw == "" { + return "https://item.taobao.com/item.htm?id=" + url.QueryEscape(itemID), nil + } + if strings.HasPrefix(raw, "//") { + raw = "https:" + raw + } + u, err := url.Parse(raw) + if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || + u.User != nil || u.Port() != "" || u.Fragment != "" || u.Opaque != "" || + u.Path != "/item.htm" || u.RawPath != "" || + (strings.ToLower(u.Host) != "item.taobao.com" && strings.ToLower(u.Host) != "detail.tmall.com") { + return "", fmt.Errorf("淘宝商品链接地址无效") + } + query, err := url.ParseQuery(u.RawQuery) + if err != nil || len(query["id"]) != 1 || query.Get("id") != itemID { + return "", fmt.Errorf("淘宝商品链接与商品 ID 不一致") + } + return raw, nil +} + func detailPageRequiresLogin(page detailPage) bool { parsed, _ := url.Parse(page.URL) host := strings.ToLower(parsed.Hostname()) diff --git a/internal/taobao/detail_test.go b/internal/taobao/detail_test.go index 660f237..841ae98 100644 --- a/internal/taobao/detail_test.go +++ b/internal/taobao/detail_test.go @@ -1,10 +1,75 @@ package taobao import ( + "context" + "errors" "reflect" "testing" + "time" ) +func TestDetailURLValidatesIdentityAndPreservesSourceParameters(t *testing.T) { + for _, tc := range []struct { + name, raw, want string + }{ + {"missing", "", "https://item.taobao.com/item.htm?id=1001"}, + {"taobao", "https://item.taobao.com/item.htm?spm=fictional&id=1001&skuId=2002", "https://item.taobao.com/item.htm?spm=fictional&id=1001&skuId=2002"}, + {"tmall", "https://detail.tmall.com/item.htm?id=1001&ns=1&xxc=fictional%2Fsource", "https://detail.tmall.com/item.htm?id=1001&ns=1&xxc=fictional%2Fsource"}, + {"protocol-relative", "//detail.tmall.com/item.htm?id=1001", "https://detail.tmall.com/item.htm?id=1001"}, + {"http", "http://item.taobao.com/item.htm?id=1001", "http://item.taobao.com/item.htm?id=1001"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := (SimilarItem{ItemID: "1001", URL: tc.raw}).DetailURL() + if err != nil || got != tc.want { + t.Fatalf("got=%s err=%v", got, err) + } + }) + } + for _, raw := range []string{ + "javascript:alert(1)", "file:///item.htm?id=1001", "/item.htm?id=1001", + "https://item.taobao.com.example.invalid/item.htm?id=1001", + "https://example.invalid/item.htm?id=1001", "https://store.taobao.com/item.htm?id=1001", + "https://fictional@item.taobao.com/item.htm?id=1001", "https://item.taobao.com:444/item.htm?id=1001", + "https://item.taobao.com/item.htm?id=1001#fictional", "https://item.taobao.com/punish?id=1001", + "https://item.taobao.com/item.htm?id=1002", "https://item.taobao.com/item.htm", + "https://item.taobao.com/item.htm?id=1001&id=1002", "https://item.taobao.com/item.htm?id=1001&spm=%zz", + } { + page := &fakePage{} + if _, err := ExtractDetailVideos(page, SimilarItem{ItemID: "1001", URL: raw}, 0); err == nil || len(page.navigations) != 0 { + t.Fatalf("invalid link must not navigate: %s, err=%v", raw, err) + } + } +} + +func TestCandidateUsesFullLinkAndRiskStillStopsBeforeNextItem(t *testing.T) { + original := "https://detail.tmall.com/item.htm?ns=1&id=1001&skuId=2002&xxc=fictional" + for _, blocked := range []bool{false, true} { + page := &fakePage{pages: map[string]detailPage{}} + page.pages[original] = detailPage{URLs: []string{"https://cloud.video.taobao.com/fictional.mp4"}} + if blocked { + page.pages[original] = detailPage{Text: "系统检测到当前访问存在异常"} + } + cache := make(DetailCache) + items := []SimilarItem{{ItemID: "1001", URL: original}, {ItemID: "1002"}} + selected, urls, err := SelectVideo(context.Background(), page, items, CandidateOptions{Limit: 2, DetailWait: time.Duration(0)}, cache) + if len(page.navigations) != 1 || page.navigations[0] != original { + t.Fatalf("must navigate original URL once: %v", page.navigations) + } + if blocked { + if !errors.Is(err, ErrAccessBlocked) || len(cache) != 0 { + t.Fatalf("risk must stop and not cache: %v", err) + } + } else { + if err != nil || len(urls) != 1 || selected.URL != original { + t.Fatalf("original link must reach selected download source: %v", err) + } + if _, _, err := SelectVideo(context.Background(), page, items, CandidateOptions{Limit: 2}, cache); err != nil || len(page.navigations) != 1 { + t.Fatalf("cached source should not navigate again: %v", err) + } + } + } +} + func Test视频地址过滤只保留淘宝视频CDN(t *testing.T) { got := filterVideoURLs([]string{ "https://cloud.video.taobao.com/play/a.mp4?token=1", diff --git a/internal/taobao/imagesearch.go b/internal/taobao/imagesearch.go index 303d586..be4d5c6 100644 --- a/internal/taobao/imagesearch.go +++ b/internal/taobao/imagesearch.go @@ -208,11 +208,12 @@ func parseImageSearchResult(responseText string) ([]SimilarItem, error) { Ret []string `json:"ret"` Data struct { Items []struct { - ItemID json.RawMessage `json:"item_id"` - ItemIDAlt json.RawMessage `json:"itemId"` - NID json.RawMessage `json:"nid"` - Title string `json:"title"` - UmpPrice struct { + AuctionURL string `json:"auctionURL"` + ItemID json.RawMessage `json:"item_id"` + ItemIDAlt json.RawMessage `json:"itemId"` + NID json.RawMessage `json:"nid"` + Title string `json:"title"` + UmpPrice struct { ItemPrice json.RawMessage `json:"item_price"` } `json:"umpPriceLog"` PriceShow struct { @@ -257,11 +258,17 @@ func parseImageSearchResult(responseText string) ([]SimilarItem, error) { } items := make([]SimilarItem, 0, len(response.Data.Items)) + invalidLinks := 0 for _, source := range response.Data.Items { id := firstJSONText(source.ItemID, source.ItemIDAlt, source.NID) if id == "" { continue } + itemURL, err := (SimilarItem{ItemID: id, URL: source.AuctionURL}).DetailURL() + if err != nil { + invalidLinks++ + continue + } image := source.PicPath if image == "" { image = source.PicURL @@ -274,9 +281,12 @@ func parseImageSearchResult(responseText string) ([]SimilarItem, error) { Image: image, Sales: jsonText(source.RealSales), Region: source.Procity, - URL: "https://item.taobao.com/item.htm?id=" + url.QueryEscape(id), + URL: itemURL, }) } + if len(items) == 0 && invalidLinks > 0 { + return nil, fmt.Errorf("淘宝以图搜返回了无效商品链接") + } return items, nil } diff --git a/internal/taobao/imagesearch_test.go b/internal/taobao/imagesearch_test.go index c27e8bc..2b1e31d 100644 --- a/internal/taobao/imagesearch_test.go +++ b/internal/taobao/imagesearch_test.go @@ -51,3 +51,23 @@ func TestParseImageSearchResultMapsFieldsAndSkipsMissingID(t *testing.T) { t.Fatalf("第三条回退字段映射不正确：%#v", got) } } + +func TestImageSearchPreservesAuctionURLAndRejectsInvalidLinks(t *testing.T) { + for _, original := range []string{ + "https://item.taobao.com/item.htm?spm=fictional.source&id=1001&skuId=2002", + "https://detail.tmall.com/item.htm?id=1001&abbucket=fictional&xxc=fictional%2Fsource", + } { + items, err := parseImageSearchResult(`{"ret":["SUCCESS::调用成功"],"data":{"itemsArray":[{"item_id":"1001","auctionURL":"` + original + `"}]}}`) + if err != nil || len(items) != 1 || items[0].URL != original { + t.Fatalf("original link not preserved: items=%v err=%v", items, err) + } + } + _, err := parseImageSearchResult(`{"ret":["SUCCESS::调用成功"],"data":{"itemsArray":[{"item_id":"1001","auctionURL":"https://example.invalid/item.htm?id=1001&secret=fictional"}]}}`) + if err == nil || strings.Contains(err.Error(), "fictional") || strings.Contains(err.Error(), "example.invalid") { + t.Fatalf("invalid link must fail without leaking URL: %v", err) + } + items, err := parseImageSearchResult(`{"ret":["SUCCESS::调用成功"],"data":{"itemsArray":[{"item_id":"1001","auctionURL":"https://example.invalid/item.htm?id=1001"},{"item_id":"1002","auctionURL":"https://item.taobao.com/item.htm?id=1002&spm=fictional"}]}}`) + if err != nil || len(items) != 1 || items[0].ItemID != "1002" { + t.Fatalf("invalid candidate must not discard valid candidates: %v, %v", items, err) + } +}