diff --git a/docs/12-syb-erp-interface.md b/docs/12-syb-erp-interface.md index 3d5e117..1f2bb0a 100644 --- a/docs/12-syb-erp-interface.md +++ b/docs/12-syb-erp-interface.md @@ -198,8 +198,10 @@ POST /am/stock/list → data.list 是数组，data.total 是当前页条 ### 4.3 分页 `[必须]` 一个跨日范围要拆成逐日查询。先逐日调用 `listTotal` 做全范围容量 -预检，确认合计不超限后，再按天以 `length = 20` 翻页调用 `list`。明细仍按 -最多 100 个货运单 ID 一批读取。 +预检，确认合计不超限后，再按天以配置的 `length` 翻页调用 `list`。2026-08-29 +对稳定历史日（753 张）只读验证：20 条在第 38 页超时；50 条 16 页用时 24.242 秒； +100 条 8 页用时 2 分 34.751 秒；200 条 4 页用时 22.183 秒。三种大页均满足总数、 +页长、唯一 ID 和尾页完整性，生产采用保守的 50；明细仍按最多 100 个 ID 一批读取。 `[必须]` **必须有单次同步的条数上限**，超了报错而不是硬拉。 Admin 默认 `max_matches = 10000`，可以在配置中调整；上限针对整个日期范围的 @@ -514,7 +516,7 @@ GoAuto 已有的做法见 `server/config/extend.go` 的 `ApplyEnvironment()`： | `GOAUTO_SYB_USERNAME` | 环境变量 | 顺云宝账号 | | `GOAUTO_SYB_PASSWORD` | 环境变量 | 顺云宝密码 | | `extend.syb.base_url` | settings.yml | 默认 `https://www.shunyunbaoerp.com` | -| `extend.syb.page_size` | settings.yml | 列表每页条数，抓包里网页用 20 | +| `extend.syb.page_size` | settings.yml | 列表每页条数；2026-08-29 只读实验验证 50/100/200 完整，生产默认采用保守值 50 | | `extend.syb.max_matches` | settings.yml | 单次同步货运单数上限，超过即停止 | | `extend.syb.ocr_url` | settings.yml | 验证码识别服务；留空 = 禁用，只走手工输入 | | `extend.syb.ocr_max_attempts` | settings.yml | OCR 重试次数，默认 5 | diff --git a/server/app/goauto/sybclient/live_test.go b/server/app/goauto/sybclient/live_test.go index 12f4cb7..24c0034 100644 --- a/server/app/goauto/sybclient/live_test.go +++ b/server/app/goauto/sybclient/live_test.go @@ -18,6 +18,7 @@ package sybclient import ( "context" + "fmt" "os" "testing" "time" @@ -120,6 +121,52 @@ func TestLiveListPagingContract(t *testing.T) { } } +func TestLiveListPageSizeCapacity(t *testing.T) { + client, _ := liveClient(t) + date := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + for _, pageSize := range []int{20, 50, 100, 200} { + t.Run(fmt.Sprintf("pageSize_%d", pageSize), func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + defer cancel() + started := time.Now() + total, err := client.ListTotal(ctx, date, date, pageSize) + if err != nil { + t.Fatalf("listTotal 失败: %v", err) + } + seen := make(map[int64]struct{}, total) + pages := 0 + for start := 0; start < total; start += pageSize { + pages++ + rows, pageTotal, err := client.ListPage(ctx, date, date, start, pages, pageSize) + if err != nil { + t.Fatalf("第 %d 页失败: %v", pages, err) + } + expected := pageSize + if remaining := total - start; remaining < expected { + expected = remaining + } + if pageTotal != len(rows) || len(rows) != expected { + t.Fatalf("第 %d 页不完整: total=%d rows=%d expected=%d", pages, pageTotal, len(rows), expected) + } + for _, row := range rows { + if row.ID <= 0 { + t.Fatal("列表包含非法 ID") + } + if _, duplicate := seen[row.ID]; duplicate { + t.Fatal("列表包含重复 ID") + } + seen[row.ID] = struct{}{} + } + } + after, err := client.ListTotal(ctx, date, date, pageSize) + if err != nil || after != total || len(seen) != total { + t.Fatalf("完整性失败: before=%d after=%d unique=%d err=%v", total, after, len(seen), err) + } + t.Logf("date=%s pageSize=%d total=%d unique=%d pages=%d elapsed=%s", date, pageSize, total, len(seen), pages, time.Since(started).Round(time.Millisecond)) + }) + } +} + // 验证明细结构和 12 §6.1「一单多商品是嵌套，不是多行」、 // §5.1「productPrice 单位是元，不是分」。 func TestLiveDetailShape(t *testing.T) { diff --git a/server/app/goauto/sybimport/sync.go b/server/app/goauto/sybimport/sync.go index 1407529..6805461 100644 --- a/server/app/goauto/sybimport/sync.go +++ b/server/app/goauto/sybimport/sync.go @@ -41,7 +41,8 @@ const ( // maxSyncDays bounds one request's window. It is a guard against a typo in // the date range turning into tens of thousands of remote reads before // MaxMatches trips. - maxSyncDays = 31 + maxSyncDays = 31 + maxTodaySnapshotAttempts = 3 ) // SyncReport summarises one sync run. @@ -77,8 +78,26 @@ type SyncProgress struct { DaysProcessed int } +var syncNow = time.Now + type ProgressFunc func(SyncProgress) error +type snapshotDriftError struct { + message string + rows []sybclient.StockRow + valid bool +} + +func (err *snapshotDriftError) Error() string { return err.message } + +func shanghaiToday() string { + location, err := time.LoadLocation("Asia/Shanghai") + if err != nil { + location = time.FixedZone("Asia/Shanghai", 8*60*60) + } + return syncNow().In(location).Format("2006-01-02") +} + // Sync pulls every shipment order in [dateFrom, dateTo] and folds each detail // line into the SYB/Shopee archive through ApplyDetail. // @@ -173,9 +192,9 @@ func SyncWithProgress(ctx context.Context, db *gorm.DB, client *sybclient.Client // day's snapshot is whole; filtering before that would let drift among // other shops' orders hide a hole in the ones we do want // (docs/12-syb-erp-interface.md §8). - rows, err := loadDailyList(ctx, client, plan.date, pageSize, plan.total) - if err != nil { - return report, err + rows, listErr := loadDailyListWithRecovery(ctx, client, plan.date, pageSize, plan.total, maxMatches) + if listErr != nil && len(rows) == 0 { + return report, listErr } report.OrderCount += len(rows) @@ -252,6 +271,9 @@ func SyncWithProgress(ctx context.Context, db *gorm.DB, client *sybclient.Client return report, err } } + if listErr != nil { + return report, listErr + } if err := emit(dayIndex + 1); err != nil { return report, err } @@ -296,6 +318,38 @@ func moveAcceptedToSkipped(report *SyncReport, listName, detailName string, allo // `[必须]` The loop bound comes from expectedTotal (listTotal), because // list.total is the current page's row count, not the filtered total (§4.3). // Driving the loop with the response's own total would stop after page one. +func loadDailyListWithRecovery(ctx context.Context, client *sybclient.Client, date string, pageSize, expectedTotal, maxMatches int) ([]sybclient.StockRow, error) { + if date != shanghaiToday() { + return loadDailyList(ctx, client, date, pageSize, expectedTotal) + } + var last *snapshotDriftError + for attempt := 1; attempt <= maxTodaySnapshotAttempts; attempt++ { + if attempt > 1 { + total, err := client.ListTotal(ctx, date, date, pageSize) + if err != nil { + return nil, fmt.Errorf("重新查询今天 %s 货运单总数失败: %w", date, err) + } + if total > maxMatches { + return nil, fmt.Errorf("今天 %s 的货运单总数 %d 超过单次同步上限 %d", date, total, maxMatches) + } + expectedTotal = total + } + rows, err := loadDailyList(ctx, client, date, pageSize, expectedTotal) + if err == nil { + return rows, nil + } + var drift *snapshotDriftError + if !errors.As(err, &drift) { + return nil, err + } + last = drift + } + if last != nil && last.valid && len(last.rows) > 0 { + return last.rows, fmt.Errorf("今天持续变化，已保存本次取得的完整明细，但未形成稳定快照；下次同步继续覆盖: %w", last) + } + return nil, fmt.Errorf("今天持续变化，连续 %d 次未形成稳定快照，本次同步停止: %w", maxTodaySnapshotAttempts, last) +} + func loadDailyList(ctx context.Context, client *sybclient.Client, date string, pageSize, expectedTotal int) ([]sybclient.StockRow, error) { rows := make([]sybclient.StockRow, 0, expectedTotal) seen := make(map[int64]struct{}, expectedTotal) @@ -311,17 +365,22 @@ func loadDailyList(ctx context.Context, client *sybclient.Client, date string, p if remaining := expectedTotal - start; remaining < pageSize { expectedPageCount = remaining } - if pageCount != expectedPageCount || len(page) != expectedPageCount { - return nil, fmt.Errorf("%s 货运单列表第 %d 页不完整：预期 %d 行，实际 %d 行；"+ - "分页期间数据发生变化，本次同步停止", date, pageIndex, expectedPageCount, len(page)) + if pageCount != len(page) { + return nil, fmt.Errorf("%s 货运单列表第 %d 页响应条数不自洽：total=%d，list=%d", date, pageIndex, pageCount, len(page)) } for _, row := range page { + if row.ID <= 0 { + return nil, fmt.Errorf("%s 货运单列表包含非法 id=%d", date, row.ID) + } if _, duplicate := seen[row.ID]; duplicate { - continue + return nil, fmt.Errorf("%s 货运单列表重复返回 id=%d", date, row.ID) } seen[row.ID] = struct{}{} rows = append(rows, row) } + if len(page) != expectedPageCount { + return nil, &snapshotDriftError{message: fmt.Sprintf("%s 货运单列表第 %d 页不完整且相对初始总数发生变化：预期 %d 行，实际 %d 行", date, pageIndex, expectedPageCount, len(page)), rows: rows, valid: len(page) > expectedPageCount} + } } // Re-read the total: if it moved while we paged, some order was inserted or @@ -331,12 +390,10 @@ func loadDailyList(ctx context.Context, client *sybclient.Client, date string, p return nil, fmt.Errorf("分页后重新查询 %s 货运单总数失败: %w", date, err) } if afterTotal != expectedTotal { - return nil, fmt.Errorf("%s 货运单总数在分页期间从 %d 变为 %d，本次同步停止", - date, expectedTotal, afterTotal) + return nil, &snapshotDriftError{message: fmt.Sprintf("%s 货运单总数在分页期间从 %d 变为 %d", date, expectedTotal, afterTotal), rows: rows, valid: len(rows) == expectedTotal} } if len(rows) != expectedTotal { - return nil, fmt.Errorf("%s 货运单列表不完整：预期 %d 张，分页后只有 %d 个唯一 ID", - date, expectedTotal, len(rows)) + return nil, fmt.Errorf("%s 货运单列表不完整：预期 %d 张，分页后只有 %d 个唯一 ID", date, expectedTotal, len(rows)) } return rows, nil } diff --git a/server/app/goauto/sybimport/sync_test.go b/server/app/goauto/sybimport/sync_test.go index 45699b1..75680ff 100644 --- a/server/app/goauto/sybimport/sync_test.go +++ b/server/app/goauto/sybimport/sync_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "go-admin/app/goauto/migrations" "go-admin/app/goauto/models" @@ -262,6 +263,39 @@ func TestSyncStopsWhenTotalDriftsDuringPaging(t *testing.T) { } } +func TestTodaySnapshotDriftRetriesFromFirstPage(t *testing.T) { + originalNow := syncNow + syncNow = func() time.Time { return time.Date(2026, 8, 29, 12, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*60*60)) } + t.Cleanup(func() { syncNow = originalNow }) + f := &fakeSYB{perDay: map[string]int{"2026-08-29": 10}, totalOverride: map[int]int{1: 11, 2: 10, 3: 10}} + rows, err := loadDailyListWithRecovery(context.Background(), newSyncClient(t, f), "2026-08-29", 10, 10, 1000) + if err != nil || len(rows) != 10 || f.listTotalCalls != 3 { + t.Fatalf("rows=%d totalCalls=%d err=%v", len(rows), f.listTotalCalls, err) + } +} + +func TestTodayThirdDriftReturnsValidLastListForDegradedSave(t *testing.T) { + originalNow := syncNow + syncNow = func() time.Time { return time.Date(2026, 8, 29, 12, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*60*60)) } + t.Cleanup(func() { syncNow = originalNow }) + f := &fakeSYB{perDay: map[string]int{"2026-08-29": 10}, totalOverride: map[int]int{1: 11, 2: 10, 3: 11, 4: 10, 5: 11}} + rows, err := loadDailyListWithRecovery(context.Background(), newSyncClient(t, f), "2026-08-29", 10, 10, 1000) + if err == nil || len(rows) != 10 || !strings.Contains(err.Error(), "已保存本次取得的完整明细") { + t.Fatalf("rows=%d err=%v", len(rows), err) + } +} + +func TestHistoricalSnapshotDriftDoesNotRetry(t *testing.T) { + originalNow := syncNow + syncNow = func() time.Time { return time.Date(2026, 8, 29, 12, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*60*60)) } + t.Cleanup(func() { syncNow = originalNow }) + f := &fakeSYB{perDay: map[string]int{"2026-08-28": 10}, totalOverride: map[int]int{1: 11}} + rows, err := loadDailyListWithRecovery(context.Background(), newSyncClient(t, f), "2026-08-28", 10, 10, 1000) + if err == nil || rows != nil || f.listTotalCalls != 1 { + t.Fatalf("rows=%v totalCalls=%d err=%v", rows, f.listTotalCalls, err) + } +} + // 某页行数少于预期同样是不完整快照。 func TestSyncStopsOnShortPage(t *testing.T) { db := newSyncTestDB(t) diff --git a/server/config/extend.go b/server/config/extend.go index d5f30bb..79e8ce1 100644 --- a/server/config/extend.go +++ b/server/config/extend.go @@ -55,7 +55,7 @@ type SYB struct { // SYBDefaults are the values used when settings.yml leaves a field blank. const ( DefaultSYBBaseURL = "https://www.shunyunbaoerp.com" - DefaultSYBPageSize = 20 + DefaultSYBPageSize = 50 DefaultSYBMaxMatches = 10000 DefaultSYBOcrMaxAttempts = 5 ) diff --git a/server/config/extend_settings_test.go b/server/config/extend_settings_test.go index a85d340..211f3ed 100644 --- a/server/config/extend_settings_test.go +++ b/server/config/extend_settings_test.go @@ -30,7 +30,7 @@ func TestSYBSettingsInRepoBindToExtendStruct(t *testing.T) { if syb.BaseURL != "https://www.shunyunbaoerp.com" { t.Fatalf("baseurl 没有绑定成功: %q", syb.BaseURL) } - if syb.PageSize != 20 { + if syb.PageSize != 50 { t.Fatalf("pagesize 没有绑定成功: %d", syb.PageSize) } if syb.MaxMatches != 10000 { diff --git a/server/config/settings.yml b/server/config/settings.yml index 6fe36bb..0533615 100644 --- a/server/config/settings.yml +++ b/server/config/settings.yml @@ -55,7 +55,7 @@ settings: # 以免凭据进 Git。 syb: baseurl: https://www.shunyunbaoerp.com - pagesize: 20 + pagesize: 50 maxmatches: 10000 # 验证码自动识别服务。留空 = 禁用，只走手工输入弹窗，不报错。 # `[必须]` 验证码图片会被发送到这个地址；换成别人运营的服务前要重新评估。