fix: 保留图搜原始商品链接用于详情导航 (#23)
This commit is contained in:
QiuSW
@@ -542,7 +542,7 @@ func (a *App) prepareVideoFetchOnPage(ctx context.Context, product store.Product
|
||||
result.SourceItem = selected.ItemID
|
||||
result.VideoCount = len(videoURLs)
|
||||
result.Directory = productDir
|
||||
referer := "https://item.taobao.com/item.htm?id=" + url.QueryEscape(selected.ItemID)
|
||||
referer := selected.URL
|
||||
downloaderClient := downloader.New()
|
||||
records := make([]store.Video, len(videoURLs))
|
||||
now := time.Now().Format("2006-01-02 15:04:05")
|
||||
|
||||
+1
-1
@@ -89,7 +89,7 @@ func TestRiskStopPreservesSQLiteProductVideosAndRemainingQueue(t *testing.T) {
|
||||
if stage == "search" {
|
||||
return nil, taobao.ErrAccessBlocked
|
||||
}
|
||||
return []taobao.SimilarItem{{ItemID: "1"}, {ItemID: "2"}}, nil
|
||||
return []taobao.SimilarItem{{ItemID: "1", URL: "https://detail.tmall.com/item.htm?ns=1&id=1&skuId=fictional"}, {ItemID: "2"}}, nil
|
||||
}
|
||||
finished := make(chan task.Progress, 1)
|
||||
guard := task.NewEmptyRiskGuard(1)
|
||||
|
||||
@@ -2,12 +2,18 @@
|
||||
generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件)
|
||||
wiki_page: Architecture-and-Code-Map
|
||||
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Architecture-and-Code-Map.-
|
||||
wiki_revision: e9367a04bc7af1ffbf956c9ba406fdd99f0df8ee
|
||||
synchronized_at: 2026-09-29T03:33:19Z
|
||||
wiki_revision: 662a06a06a45623d4adb68c5bf1fe86ee70103ee
|
||||
synchronized_at: 2026-09-29T07:39:06Z
|
||||
<!-- gitea-wiki-mirror:end -->
|
||||
|
||||
# 架构与代码地图
|
||||
|
||||
## 图搜原始商品链接（2026-09-29）
|
||||
|
||||
`parseImageSearchResult` 读取已通过真实响应确认的 `itemsArray[].auctionURL`，写入 `SimilarItem.URL`。`SimilarItem.DetailURL` 在 Go 侧校验 http/https、精确商品域名 `item.taobao.com` / `detail.tmall.com`、`/item.htm` 路径及唯一且匹配的 `id`，拒绝用户信息、其他端口、片段、错误编码和不可信站点；协议相对地址补 https。保留原链接查询参数与顺序，不记录完整链接或参数值。只有响应缺少链接时兼容原 ID 拼接方式；非法原链接候选跳过，不访问也不改拼旧地址，其他合法候选照常返回；若有非法链接且全部候选都不可用，则明确报错，不能记为正常无结果。
|
||||
|
||||
`SelectVideo` 校验链接后传整个 `SimilarItem` 给 `ExtractDetailVideos`，详情导航使用原链接；选中的 `SimilarItem.URL` 同时用于 CDN 下载 Referer。原有串行候选、同批 ID 缓存、登录/访问异常停止门和 SQLite 状态保持不变。这里仍是浏览器直接导航到返回的商品链接：MTOP 搜索没有生成淘宝搜索结果页，也没有模拟结果卡片点击或为详情导航额外设置 Referer。
|
||||
|
||||
## 淘宝访问异常停止与候选复用（2026-09-29）
|
||||
|
||||
`App.prepareVideoFetch` 建立专属 Chrome 连接；`prepareVideoFetchOnPage` 调用本批 `taobao.LoginGuard.Check`：首次实际处理调用 `CheckLogin` 打开「我的淘宝」深度检查，后续商品调用 `CheckCurrentLogin` 读取当前 URL/标题/有限正文和 Cookie 名称，不发送检查导航。图搜首页、MTOP 返回和详情页中的明确访问异常均映射到任务全局停止门，签名算法及请求参数不变。
|
||||
|
||||
@@ -2,12 +2,18 @@
|
||||
generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件)
|
||||
wiki_page: Business-Rules-and-Glossary
|
||||
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Business-Rules-and-Glossary.-
|
||||
wiki_revision: 6fa4581023a61ca0d5479cf46d696688b4d266f3
|
||||
synchronized_at: 2026-09-29T03:33:20Z
|
||||
wiki_revision: f4dcb37885891ef200f0d4703d97d623fa8aac47
|
||||
synchronized_at: 2026-09-29T07:39:09Z
|
||||
<!-- gitea-wiki-mirror:end -->
|
||||
|
||||
# 业务规则与术语
|
||||
|
||||
## 同款商品链接使用（2026-09-29）
|
||||
|
||||
MTOP 搜索方式、签名与请求参数保持现状。商品原链接以实际返回字段 `auctionURL` 为准，保留查询参数；允许淘宝/天猫的商品详情 `/item.htm`，必须与候选商品 ID 一致。非法链接候选跳过，不导航、不改拼旧地址、不泄漏到错误日志；其他合法候选仍可处理，全为非法链接则明确报错，不能记为正常无结果。缺少字段时才使用原 ID 拼接链接。完整链接仅用于当前查询结果、详情导航与下载 Referer，不新增持久化任务状态。
|
||||
|
||||
使用返回链接直接导航不等同于从淘宝结果页点击。当前未获得可直接打开的结果页地址证据，不猜测构造结果页/来源参数，不自动收藏商品，不承诺解除访问异常。访问异常、登录失效仍即时停批，不自动重试或恢复轮询。
|
||||
|
||||
## 淘宝访问与停止规则（2026-09-29）
|
||||
|
||||
- 明确访问异常：页面出现「当前访问存在异常」「恢复正常访问方式」「访问受限」、安全验证/验证码或 `/punish` 路径时立即停批，`stopReason=risk_blocked`。即使 Cookie 齐全或页面残留视频 URL，也不继续访问。
|
||||
|
||||
@@ -2,12 +2,18 @@
|
||||
generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件)
|
||||
wiki_page: Local-Development-and-Verification
|
||||
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Local-Development-and-Verification.-
|
||||
wiki_revision: 5027339b211f8d7762449b4e579a4886a2e8bff4
|
||||
synchronized_at: 2026-09-29T03:33:20Z
|
||||
wiki_revision: a99cdc64887622e4127aebd0693d37fffaa3f0c4
|
||||
synchronized_at: 2026-09-29T07:39:11Z
|
||||
<!-- gitea-wiki-mirror:end -->
|
||||
|
||||
# 本地开发与验证
|
||||
|
||||
## 商品原链接验证（2026-09-29）
|
||||
|
||||
`internal/taobao/imagesearch_test.go` 验证 `auctionURL` 参数保留、缺失字段兼容、非法链接候选跳过及全部非法时明确报错；`detail_test.go` 验证淘宝/天猫域名、商品 ID 匹配、拒绝恶意域名/重复 ID/错误编码/用户信息/其他端口及片段、原链接导航、缓存复用和受限后停止。`app_video_test.go` 的 SQLite 停止场景包含天猫原链接，验证商品、视频、上传及剩余队列状态保留。常规测试不访问淘宝。
|
||||
|
||||
从根目录执行 `go test ./...`、`go test -race ./...`、`go vet ./...`；Windows 构建可用 `& "$env:USERPROFILE\go\bin\wails.exe" build -o cmsp23.exe`。不要与正在更新前端嵌入资源的构建同时运行 Go 测试。真实联调须先确认专属浏览器归属、登录检查通过且没有进行中的下载任务；只读 SQLite，响应只分析必要字段名/链接结构，不落盘原始响应与凭据。遇到请求或访问异常即停止，不为验证重复请求。
|
||||
|
||||
## 淘宝风控停止门验证（2026-09-29）
|
||||
|
||||
从仓库根目录先完成前端资源构建，再执行嵌入这些资源的 Go 测试，避免两者同时更新/读取 `frontend/dist`：
|
||||
|
||||
@@ -2,12 +2,18 @@
|
||||
generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件)
|
||||
wiki_page: Troubleshooting
|
||||
wiki_url: https://git.ilapage.cn/chengma/cmsp/wiki/Troubleshooting
|
||||
wiki_revision: 8e56ccb600bd56584ca513fe87bd2cd05f4328da
|
||||
synchronized_at: 2026-09-29T03:33:21Z
|
||||
wiki_revision: c91a79baab19786e491b46e8aceb2308a2ea49d5
|
||||
synchronized_at: 2026-09-29T07:39:14Z
|
||||
<!-- gitea-wiki-mirror:end -->
|
||||
|
||||
# 故障排查
|
||||
|
||||
## 图搜成功但商品链接无法使用（2026-09-29）
|
||||
|
||||
程序优先使用 MTOP 返回的 `auctionURL`，不会丢弃原查询参数后重新按 ID 拼接。单个非法链接候选会跳过，不影响其他合法候选；如果所有候选都不可用，报“淘宝以图搜返回了无效商品链接”，检查经过脱敏的字段结构是否发生变化；只允许淘宝/天猫商品域名、`/item.htm` 和与候选匹配的唯一 `id`。不能关闭校验、盲信广告跳转域名或把完整响应/参数写入日志。缺少链接字段的旧响应仍兼容按 ID 打开。
|
||||
|
||||
保留原链接是进入路径调整，不代表浏览器已经处于淘宝搜索结果页，也不能证明平台风控缓解。受限提示仍按下方停止门处理。仅确认商品链接成功打开不足以证明视频属于当前商品或下载完成。
|
||||
|
||||
## 淘宝提示「当前访问存在异常」（2026-09-29）
|
||||
|
||||
先停止自动任务。程序检测到此提示或安全验证时应显示「淘宝访问受限」，整批停止并保留断点；不会将受限详情记为无视频，也不会自动重试。不要连续点击启动。
|
||||
|
||||
@@ -126,7 +126,7 @@ func TestAccessPromptStopsDespiteCookiesAndVideo(t *testing.T) {
|
||||
}
|
||||
for _, url := range []string{"https://item.taobao.com/punish?fictional=secret", "https://login.taobao.com/member/login.jhtml"} {
|
||||
p := &fakePage{pages: map[string]detailPage{"https://item.taobao.com/item.htm?id=1": {URL: url}}}
|
||||
_, err := ExtractDetailVideos(p, "1", 0)
|
||||
_, err := ExtractDetailVideos(p, SimilarItem{ItemID: "1"}, 0)
|
||||
want := ErrLoginInvalid
|
||||
if strings.Contains(url, "punish") {
|
||||
want = ErrAccessBlocked
|
||||
|
||||
@@ -35,6 +35,14 @@ func SelectVideo(ctx context.Context, page PageSession, items []SimilarItem, opt
|
||||
}
|
||||
seen[item.ItemID] = true
|
||||
checked++
|
||||
itemURL, err := item.DetailURL()
|
||||
if err != nil {
|
||||
if firstError == nil {
|
||||
firstError = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
item.URL = itemURL
|
||||
urls, cached := cache[item.ItemID]
|
||||
if !cached {
|
||||
if visited {
|
||||
@@ -50,7 +58,7 @@ func SelectVideo(ctx context.Context, page PageSession, items []SimilarItem, opt
|
||||
return SimilarItem{}, nil, err
|
||||
}
|
||||
visited = true
|
||||
detail, err := ExtractDetailVideos(page, item.ItemID, options.DetailWait)
|
||||
detail, err := ExtractDetailVideos(page, item, options.DetailWait)
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return SimilarItem{}, nil, canceled
|
||||
}
|
||||
|
||||
@@ -42,12 +42,12 @@ type DetailResult struct {
|
||||
}
|
||||
|
||||
// ExtractDetailVideos 打开一个淘宝商品详情页并提取可信的视频 CDN 地址。
|
||||
func ExtractDetailVideos(cdp PageSession, itemID string, wait time.Duration) (DetailResult, error) {
|
||||
itemID = strings.TrimSpace(itemID)
|
||||
if itemID == "" {
|
||||
return DetailResult{}, fmt.Errorf("淘宝同款商品 ID 不能为空")
|
||||
func ExtractDetailVideos(cdp PageSession, item SimilarItem, wait time.Duration) (DetailResult, error) {
|
||||
itemURL, err := item.DetailURL()
|
||||
if err != nil {
|
||||
return DetailResult{}, err
|
||||
}
|
||||
itemURL := "https://item.taobao.com/item.htm?id=" + url.QueryEscape(itemID)
|
||||
itemID := item.ItemID
|
||||
if err := cdp.Navigate(itemURL, wait); err != nil {
|
||||
return DetailResult{}, fmt.Errorf("打开淘宝商品 %s 详情页失败：%w", itemID, err)
|
||||
}
|
||||
@@ -68,6 +68,34 @@ func ExtractDetailVideos(cdp PageSession, itemID string, wait time.Duration) (De
|
||||
return DetailResult{Title: page.Title, Text: page.Text, URLs: filterVideoURLs(page.URLs)}, nil
|
||||
}
|
||||
|
||||
// DetailURL 保留实际图搜返回的链接参数；仅旧响应缺少链接时按 ID 兼容。
|
||||
// 校验发生在 Go 侧，不能让外部链接把专属登录会话引向其他站点或商品。
|
||||
func (item SimilarItem) DetailURL() (string, error) {
|
||||
itemID := strings.TrimSpace(item.ItemID)
|
||||
if itemID == "" {
|
||||
return "", fmt.Errorf("淘宝同款商品 ID 不能为空")
|
||||
}
|
||||
raw := strings.TrimSpace(item.URL)
|
||||
if raw == "" {
|
||||
return "https://item.taobao.com/item.htm?id=" + url.QueryEscape(itemID), nil
|
||||
}
|
||||
if strings.HasPrefix(raw, "//") {
|
||||
raw = "https:" + raw
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") ||
|
||||
u.User != nil || u.Port() != "" || u.Fragment != "" || u.Opaque != "" ||
|
||||
u.Path != "/item.htm" || u.RawPath != "" ||
|
||||
(strings.ToLower(u.Host) != "item.taobao.com" && strings.ToLower(u.Host) != "detail.tmall.com") {
|
||||
return "", fmt.Errorf("淘宝商品链接地址无效")
|
||||
}
|
||||
query, err := url.ParseQuery(u.RawQuery)
|
||||
if err != nil || len(query["id"]) != 1 || query.Get("id") != itemID {
|
||||
return "", fmt.Errorf("淘宝商品链接与商品 ID 不一致")
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
func detailPageRequiresLogin(page detailPage) bool {
|
||||
parsed, _ := url.Parse(page.URL)
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
|
||||
@@ -1,10 +1,75 @@
|
||||
package taobao
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDetailURLValidatesIdentityAndPreservesSourceParameters(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, raw, want string
|
||||
}{
|
||||
{"missing", "", "https://item.taobao.com/item.htm?id=1001"},
|
||||
{"taobao", "https://item.taobao.com/item.htm?spm=fictional&id=1001&skuId=2002", "https://item.taobao.com/item.htm?spm=fictional&id=1001&skuId=2002"},
|
||||
{"tmall", "https://detail.tmall.com/item.htm?id=1001&ns=1&xxc=fictional%2Fsource", "https://detail.tmall.com/item.htm?id=1001&ns=1&xxc=fictional%2Fsource"},
|
||||
{"protocol-relative", "//detail.tmall.com/item.htm?id=1001", "https://detail.tmall.com/item.htm?id=1001"},
|
||||
{"http", "http://item.taobao.com/item.htm?id=1001", "http://item.taobao.com/item.htm?id=1001"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := (SimilarItem{ItemID: "1001", URL: tc.raw}).DetailURL()
|
||||
if err != nil || got != tc.want {
|
||||
t.Fatalf("got=%s err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, raw := range []string{
|
||||
"javascript:alert(1)", "file:///item.htm?id=1001", "/item.htm?id=1001",
|
||||
"https://item.taobao.com.example.invalid/item.htm?id=1001",
|
||||
"https://example.invalid/item.htm?id=1001", "https://store.taobao.com/item.htm?id=1001",
|
||||
"https://fictional@item.taobao.com/item.htm?id=1001", "https://item.taobao.com:444/item.htm?id=1001",
|
||||
"https://item.taobao.com/item.htm?id=1001#fictional", "https://item.taobao.com/punish?id=1001",
|
||||
"https://item.taobao.com/item.htm?id=1002", "https://item.taobao.com/item.htm",
|
||||
"https://item.taobao.com/item.htm?id=1001&id=1002", "https://item.taobao.com/item.htm?id=1001&spm=%zz",
|
||||
} {
|
||||
page := &fakePage{}
|
||||
if _, err := ExtractDetailVideos(page, SimilarItem{ItemID: "1001", URL: raw}, 0); err == nil || len(page.navigations) != 0 {
|
||||
t.Fatalf("invalid link must not navigate: %s, err=%v", raw, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidateUsesFullLinkAndRiskStillStopsBeforeNextItem(t *testing.T) {
|
||||
original := "https://detail.tmall.com/item.htm?ns=1&id=1001&skuId=2002&xxc=fictional"
|
||||
for _, blocked := range []bool{false, true} {
|
||||
page := &fakePage{pages: map[string]detailPage{}}
|
||||
page.pages[original] = detailPage{URLs: []string{"https://cloud.video.taobao.com/fictional.mp4"}}
|
||||
if blocked {
|
||||
page.pages[original] = detailPage{Text: "系统检测到当前访问存在异常"}
|
||||
}
|
||||
cache := make(DetailCache)
|
||||
items := []SimilarItem{{ItemID: "1001", URL: original}, {ItemID: "1002"}}
|
||||
selected, urls, err := SelectVideo(context.Background(), page, items, CandidateOptions{Limit: 2, DetailWait: time.Duration(0)}, cache)
|
||||
if len(page.navigations) != 1 || page.navigations[0] != original {
|
||||
t.Fatalf("must navigate original URL once: %v", page.navigations)
|
||||
}
|
||||
if blocked {
|
||||
if !errors.Is(err, ErrAccessBlocked) || len(cache) != 0 {
|
||||
t.Fatalf("risk must stop and not cache: %v", err)
|
||||
}
|
||||
} else {
|
||||
if err != nil || len(urls) != 1 || selected.URL != original {
|
||||
t.Fatalf("original link must reach selected download source: %v", err)
|
||||
}
|
||||
if _, _, err := SelectVideo(context.Background(), page, items, CandidateOptions{Limit: 2}, cache); err != nil || len(page.navigations) != 1 {
|
||||
t.Fatalf("cached source should not navigate again: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test视频地址过滤只保留淘宝视频CDN(t *testing.T) {
|
||||
got := filterVideoURLs([]string{
|
||||
"https://cloud.video.taobao.com/play/a.mp4?token=1",
|
||||
|
||||
@@ -208,11 +208,12 @@ func parseImageSearchResult(responseText string) ([]SimilarItem, error) {
|
||||
Ret []string `json:"ret"`
|
||||
Data struct {
|
||||
Items []struct {
|
||||
ItemID json.RawMessage `json:"item_id"`
|
||||
ItemIDAlt json.RawMessage `json:"itemId"`
|
||||
NID json.RawMessage `json:"nid"`
|
||||
Title string `json:"title"`
|
||||
UmpPrice struct {
|
||||
AuctionURL string `json:"auctionURL"`
|
||||
ItemID json.RawMessage `json:"item_id"`
|
||||
ItemIDAlt json.RawMessage `json:"itemId"`
|
||||
NID json.RawMessage `json:"nid"`
|
||||
Title string `json:"title"`
|
||||
UmpPrice struct {
|
||||
ItemPrice json.RawMessage `json:"item_price"`
|
||||
} `json:"umpPriceLog"`
|
||||
PriceShow struct {
|
||||
@@ -257,11 +258,17 @@ func parseImageSearchResult(responseText string) ([]SimilarItem, error) {
|
||||
}
|
||||
|
||||
items := make([]SimilarItem, 0, len(response.Data.Items))
|
||||
invalidLinks := 0
|
||||
for _, source := range response.Data.Items {
|
||||
id := firstJSONText(source.ItemID, source.ItemIDAlt, source.NID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
itemURL, err := (SimilarItem{ItemID: id, URL: source.AuctionURL}).DetailURL()
|
||||
if err != nil {
|
||||
invalidLinks++
|
||||
continue
|
||||
}
|
||||
image := source.PicPath
|
||||
if image == "" {
|
||||
image = source.PicURL
|
||||
@@ -274,9 +281,12 @@ func parseImageSearchResult(responseText string) ([]SimilarItem, error) {
|
||||
Image: image,
|
||||
Sales: jsonText(source.RealSales),
|
||||
Region: source.Procity,
|
||||
URL: "https://item.taobao.com/item.htm?id=" + url.QueryEscape(id),
|
||||
URL: itemURL,
|
||||
})
|
||||
}
|
||||
if len(items) == 0 && invalidLinks > 0 {
|
||||
return nil, fmt.Errorf("淘宝以图搜返回了无效商品链接")
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -51,3 +51,23 @@ func TestParseImageSearchResultMapsFieldsAndSkipsMissingID(t *testing.T) {
|
||||
t.Fatalf("第三条回退字段映射不正确：%#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageSearchPreservesAuctionURLAndRejectsInvalidLinks(t *testing.T) {
|
||||
for _, original := range []string{
|
||||
"https://item.taobao.com/item.htm?spm=fictional.source&id=1001&skuId=2002",
|
||||
"https://detail.tmall.com/item.htm?id=1001&abbucket=fictional&xxc=fictional%2Fsource",
|
||||
} {
|
||||
items, err := parseImageSearchResult(`{"ret":["SUCCESS::调用成功"],"data":{"itemsArray":[{"item_id":"1001","auctionURL":"` + original + `"}]}}`)
|
||||
if err != nil || len(items) != 1 || items[0].URL != original {
|
||||
t.Fatalf("original link not preserved: items=%v err=%v", items, err)
|
||||
}
|
||||
}
|
||||
_, err := parseImageSearchResult(`{"ret":["SUCCESS::调用成功"],"data":{"itemsArray":[{"item_id":"1001","auctionURL":"https://example.invalid/item.htm?id=1001&secret=fictional"}]}}`)
|
||||
if err == nil || strings.Contains(err.Error(), "fictional") || strings.Contains(err.Error(), "example.invalid") {
|
||||
t.Fatalf("invalid link must fail without leaking URL: %v", err)
|
||||
}
|
||||
items, err := parseImageSearchResult(`{"ret":["SUCCESS::调用成功"],"data":{"itemsArray":[{"item_id":"1001","auctionURL":"https://example.invalid/item.htm?id=1001"},{"item_id":"1002","auctionURL":"https://item.taobao.com/item.htm?id=1002&spm=fictional"}]}}`)
|
||||
if err != nil || len(items) != 1 || items[0].ItemID != "1002" {
|
||||
t.Fatalf("invalid candidate must not discard valid candidates: %v, %v", items, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user