OPC/goauto
2
0
Fork 0
Code Issues 196 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(#145): stabilize SYB daily pagination

Browse Source
This commit is contained in:
QiuSW
2026-08-29 18:08:42 +08:00
parent 8afbdbc8ff
commit 7afe5ae9ce
7 changed files with 158 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/goauto/sybclient/live_test.go
+47
View File
@@ -18,6 +18,7 @@ package sybclient
import (
"context"
"fmt"
"os"
"testing"
"time"
@@ -120,6 +121,52 @@ func TestLiveListPagingContract(t *testing.T) {
}
}
func TestLiveListPageSizeCapacity(t *testing.T) {
client, _ := liveClient(t)
date := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
for _, pageSize := range []int{20, 50, 100, 200} {
t.Run(fmt.Sprintf("pageSize_%d", pageSize), func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
defer cancel()
started := time.Now()
total, err := client.ListTotal(ctx, date, date, pageSize)
if err != nil {
t.Fatalf("listTotal 失败: %v", err)
}
seen := make(map[int64]struct{}, total)
pages := 0
for start := 0; start < total; start += pageSize {
pages++
rows, pageTotal, err := client.ListPage(ctx, date, date, start, pages, pageSize)
if err != nil {
t.Fatalf("第 %d 页失败: %v", pages, err)
}
expected := pageSize
if remaining := total - start; remaining < expected {
expected = remaining
}
if pageTotal != len(rows) || len(rows) != expected {
t.Fatalf("第 %d 页不完整: total=%d rows=%d expected=%d", pages, pageTotal, len(rows), expected)
}
for _, row := range rows {
if row.ID <= 0 {
t.Fatal("列表包含非法 ID")
}
if _, duplicate := seen[row.ID]; duplicate {
t.Fatal("列表包含重复 ID")
}
seen[row.ID] = struct{}{}
}
}
after, err := client.ListTotal(ctx, date, date, pageSize)
if err != nil || after != total || len(seen) != total {
t.Fatalf("完整性失败: before=%d after=%d unique=%d err=%v", total, after, len(seen), err)
}
t.Logf("date=%s pageSize=%d total=%d unique=%d pages=%d elapsed=%s", date, pageSize, total, len(seen), pages, time.Since(started).Round(time.Millisecond))
})
}
}
// 验证明细结构和 12 §6.1「一单多商品是嵌套，不是多行」、
// §5.1「productPrice 单位是元，不是分」。
func TestLiveDetailShape(t *testing.T) {
server/app/goauto/sybimport/sync.go
+69 -12
View File
@@ -41,7 +41,8 @@ const (
// maxSyncDays bounds one request's window. It is a guard against a typo in
// the date range turning into tens of thousands of remote reads before
// MaxMatches trips.
maxSyncDays = 31
maxSyncDays = 31
maxTodaySnapshotAttempts = 3
)
// SyncReport summarises one sync run.
@@ -77,8 +78,26 @@ type SyncProgress struct {
DaysProcessed int
}
var syncNow = time.Now
type ProgressFunc func(SyncProgress) error
type snapshotDriftError struct {
message string
rows []sybclient.StockRow
valid bool
}
func (err *snapshotDriftError) Error() string { return err.message }
func shanghaiToday() string {
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
location = time.FixedZone("Asia/Shanghai", 8*60*60)
}
return syncNow().In(location).Format("2006-01-02")
}
// Sync pulls every shipment order in [dateFrom, dateTo] and folds each detail
// line into the SYB/Shopee archive through ApplyDetail.
//
@@ -173,9 +192,9 @@ func SyncWithProgress(ctx context.Context, db *gorm.DB, client *sybclient.Client
// day's snapshot is whole; filtering before that would let drift among
// other shops' orders hide a hole in the ones we do want
// (docs/12-syb-erp-interface.md §8).
rows, err := loadDailyList(ctx, client, plan.date, pageSize, plan.total)
if err != nil {
return report, err
rows, listErr := loadDailyListWithRecovery(ctx, client, plan.date, pageSize, plan.total, maxMatches)
if listErr != nil && len(rows) == 0 {
return report, listErr
}
report.OrderCount += len(rows)
@@ -252,6 +271,9 @@ func SyncWithProgress(ctx context.Context, db *gorm.DB, client *sybclient.Client
return report, err
}
}
if listErr != nil {
return report, listErr
}
if err := emit(dayIndex + 1); err != nil {
return report, err
}
@@ -296,6 +318,38 @@ func moveAcceptedToSkipped(report *SyncReport, listName, detailName string, allo
// `[必须]` The loop bound comes from expectedTotal (listTotal), because
// list.total is the current page's row count, not the filtered total (§4.3).
// Driving the loop with the response's own total would stop after page one.
func loadDailyListWithRecovery(ctx context.Context, client *sybclient.Client, date string, pageSize, expectedTotal, maxMatches int) ([]sybclient.StockRow, error) {
if date != shanghaiToday() {
return loadDailyList(ctx, client, date, pageSize, expectedTotal)
}
var last *snapshotDriftError
for attempt := 1; attempt <= maxTodaySnapshotAttempts; attempt++ {
if attempt > 1 {
total, err := client.ListTotal(ctx, date, date, pageSize)
if err != nil {
return nil, fmt.Errorf("重新查询今天 %s 货运单总数失败: %w", date, err)
}
if total > maxMatches {
return nil, fmt.Errorf("今天 %s 的货运单总数 %d 超过单次同步上限 %d", date, total, maxMatches)
}
expectedTotal = total
}
rows, err := loadDailyList(ctx, client, date, pageSize, expectedTotal)
if err == nil {
return rows, nil
}
var drift *snapshotDriftError
if !errors.As(err, &drift) {
return nil, err
}
last = drift
}
if last != nil && last.valid && len(last.rows) > 0 {
return last.rows, fmt.Errorf("今天持续变化，已保存本次取得的完整明细，但未形成稳定快照；下次同步继续覆盖: %w", last)
}
return nil, fmt.Errorf("今天持续变化，连续 %d 次未形成稳定快照，本次同步停止: %w", maxTodaySnapshotAttempts, last)
}
func loadDailyList(ctx context.Context, client *sybclient.Client, date string, pageSize, expectedTotal int) ([]sybclient.StockRow, error) {
rows := make([]sybclient.StockRow, 0, expectedTotal)
seen := make(map[int64]struct{}, expectedTotal)
@@ -311,17 +365,22 @@ func loadDailyList(ctx context.Context, client *sybclient.Client, date string, p
if remaining := expectedTotal - start; remaining < pageSize {
expectedPageCount = remaining
}
if pageCount != expectedPageCount || len(page) != expectedPageCount {
return nil, fmt.Errorf("%s 货运单列表第 %d 页不完整：预期 %d 行，实际 %d 行；"+
"分页期间数据发生变化，本次同步停止", date, pageIndex, expectedPageCount, len(page))
if pageCount != len(page) {
return nil, fmt.Errorf("%s 货运单列表第 %d 页响应条数不自洽：total=%d，list=%d", date, pageIndex, pageCount, len(page))
}
for _, row := range page {
if row.ID <= 0 {
return nil, fmt.Errorf("%s 货运单列表包含非法 id=%d", date, row.ID)
}
if _, duplicate := seen[row.ID]; duplicate {
continue
return nil, fmt.Errorf("%s 货运单列表重复返回 id=%d", date, row.ID)
}
seen[row.ID] = struct{}{}
rows = append(rows, row)
}
if len(page) != expectedPageCount {
return nil, &snapshotDriftError{message: fmt.Sprintf("%s 货运单列表第 %d 页不完整且相对初始总数发生变化：预期 %d 行，实际 %d 行", date, pageIndex, expectedPageCount, len(page)), rows: rows, valid: len(page) > expectedPageCount}
}
}
// Re-read the total: if it moved while we paged, some order was inserted or
@@ -331,12 +390,10 @@ func loadDailyList(ctx context.Context, client *sybclient.Client, date string, p
return nil, fmt.Errorf("分页后重新查询 %s 货运单总数失败: %w", date, err)
}
if afterTotal != expectedTotal {
return nil, fmt.Errorf("%s 货运单总数在分页期间从 %d 变为 %d，本次同步停止",
date, expectedTotal, afterTotal)
return nil, &snapshotDriftError{message: fmt.Sprintf("%s 货运单总数在分页期间从 %d 变为 %d", date, expectedTotal, afterTotal), rows: rows, valid: len(rows) == expectedTotal}
}
if len(rows) != expectedTotal {
return nil, fmt.Errorf("%s 货运单列表不完整：预期 %d 张，分页后只有 %d 个唯一 ID",
date, expectedTotal, len(rows))
return nil, fmt.Errorf("%s 货运单列表不完整：预期 %d 张，分页后只有 %d 个唯一 ID", date, expectedTotal, len(rows))
}
return rows, nil
}
server/app/goauto/sybimport/sync_test.go
+34
View File
@@ -9,6 +9,7 @@ import (
"net/http/httptest"
"strings"
"testing"
"time"
"go-admin/app/goauto/migrations"
"go-admin/app/goauto/models"
@@ -262,6 +263,39 @@ func TestSyncStopsWhenTotalDriftsDuringPaging(t *testing.T) {
}
}
func TestTodaySnapshotDriftRetriesFromFirstPage(t *testing.T) {
originalNow := syncNow
syncNow = func() time.Time { return time.Date(2026, 8, 29, 12, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*60*60)) }
t.Cleanup(func() { syncNow = originalNow })
f := &fakeSYB{perDay: map[string]int{"2026-08-29": 10}, totalOverride: map[int]int{1: 11, 2: 10, 3: 10}}
rows, err := loadDailyListWithRecovery(context.Background(), newSyncClient(t, f), "2026-08-29", 10, 10, 1000)
if err != nil || len(rows) != 10 || f.listTotalCalls != 3 {
t.Fatalf("rows=%d totalCalls=%d err=%v", len(rows), f.listTotalCalls, err)
}
}
func TestTodayThirdDriftReturnsValidLastListForDegradedSave(t *testing.T) {
originalNow := syncNow
syncNow = func() time.Time { return time.Date(2026, 8, 29, 12, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*60*60)) }
t.Cleanup(func() { syncNow = originalNow })
f := &fakeSYB{perDay: map[string]int{"2026-08-29": 10}, totalOverride: map[int]int{1: 11, 2: 10, 3: 11, 4: 10, 5: 11}}
rows, err := loadDailyListWithRecovery(context.Background(), newSyncClient(t, f), "2026-08-29", 10, 10, 1000)
if err == nil || len(rows) != 10 || !strings.Contains(err.Error(), "已保存本次取得的完整明细") {
t.Fatalf("rows=%d err=%v", len(rows), err)
}
}
func TestHistoricalSnapshotDriftDoesNotRetry(t *testing.T) {
originalNow := syncNow
syncNow = func() time.Time { return time.Date(2026, 8, 29, 12, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*60*60)) }
t.Cleanup(func() { syncNow = originalNow })
f := &fakeSYB{perDay: map[string]int{"2026-08-28": 10}, totalOverride: map[int]int{1: 11}}
rows, err := loadDailyListWithRecovery(context.Background(), newSyncClient(t, f), "2026-08-28", 10, 10, 1000)
if err == nil || rows != nil || f.listTotalCalls != 1 {
t.Fatalf("rows=%v totalCalls=%d err=%v", rows, f.listTotalCalls, err)
}
}
// 某页行数少于预期同样是不完整快照。
func TestSyncStopsOnShortPage(t *testing.T) {
db := newSyncTestDB(t)
server/config/extend.go
+1 -1
View File
@@ -55,7 +55,7 @@ type SYB struct {
// SYBDefaults are the values used when settings.yml leaves a field blank.
const (
DefaultSYBBaseURL = "https://www.shunyunbaoerp.com"
DefaultSYBPageSize = 20
DefaultSYBPageSize = 50
DefaultSYBMaxMatches = 10000
DefaultSYBOcrMaxAttempts = 5
)
server/config/extend_settings_test.go
+1 -1
View File
@@ -30,7 +30,7 @@ func TestSYBSettingsInRepoBindToExtendStruct(t *testing.T) {
if syb.BaseURL != "https://www.shunyunbaoerp.com" {
t.Fatalf("baseurl 没有绑定成功: %q", syb.BaseURL)
}
if syb.PageSize != 20 {
if syb.PageSize != 50 {
t.Fatalf("pagesize 没有绑定成功: %d", syb.PageSize)
}
if syb.MaxMatches != 10000 {
server/config/settings.yml
+1 -1
View File
@@ -55,7 +55,7 @@ settings:
# 以免凭据进 Git。
syb:
baseurl: https://www.shunyunbaoerp.com
pagesize: 20
pagesize: 50
maxmatches: 10000
# 验证码自动识别服务。留空 = 禁用，只走手工输入弹窗，不报错。
# `[必须]` 验证码图片会被发送到这个地址；换成别人运营的服务前要重新评估。