diff --git a/docs/02-architecture-and-code-map.md b/docs/02-architecture-and-code-map.md index 77edb47..925ce1d 100644 --- a/docs/02-architecture-and-code-map.md +++ b/docs/02-architecture-and-code-map.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Architecture-and-Code-Map wiki_url: https://git.ilapage.cn/OPC/goauto/wiki/Architecture-and-Code-Map.- -wiki_revision: 945d51155a72ea5b41f600262590cf928de3f5ab -synchronized_at: 2026-08-19T08:15:27Z +wiki_revision: a94218dc2c7cbf1c5d56af808a02bf65aaa19964 +synchronized_at: 2026-08-19T08:53:52Z # 架构与代码地图 @@ -81,7 +81,8 @@ Android Portal/Agent | PDD 商品档案增量迁移 | `server/cmd/migrate/migration/version-local/1786700500000_pdd_product_archive.go` | | 虾皮商品档案与规格映射 | `server/app/goauto/shopeeproduct/`（服务端 API 已实现；Admin 页面待实现） | | 虾皮商品档案增量迁移 | `server/cmd/migrate/migration/version-local/1786700600000_shopee_product_archive.go` | -| SYB 商品明细导入、解析与虾皮档案合并 | `server/app/goauto/sybimport/`（解析、幂等落库、管理端 API 已实现；SYB 接口拉取客户端未实现，无凭据；Admin 页面已实现） | +| SYB 商品明细导入、解析与虾皮档案合并 | `server/app/goauto/sybimport/`（解析、幂等落库、同步编排、导入端点、管理端 API 和 Admin 页面均已实现） | +| 顺云宝（SYB）ERP HTTP 客户端与登录会话 | `server/app/goauto/sybclient/`（登录、OCR 验证码、会话缓存、列表与明细读取；见 [SYB-ERP-Interface-Contract](SYB-ERP-Interface-Contract)） | | SYB 商品明细增量迁移 | `server/cmd/migrate/migration/version-local/1786700700000_syb_product_import.go` | | 任务领取、结果、重置与删除 | `server/app/goauto/task/` | | 管理端基线 | `web/`（go-admin-ui v3.0.0） | diff --git a/server/app/goauto/models/schema.go b/server/app/goauto/models/schema.go index fa6065c..06aecbc 100644 --- a/server/app/goauto/models/schema.go +++ b/server/app/goauto/models/schema.go @@ -303,6 +303,12 @@ type SYBSession struct { // Username is the SYB account this session belongs to. It is the business // key: re-logging in as the same account overwrites the same row. Username string `json:"username" gorm:"size:128;not null;uniqueIndex:ux_syb_session_username"` + // UserID is SYB's own numeric account id, returned by login. It must be + // cached alongside the cookies: session validation calls + // /am/user/get?id=, and that endpoint rejects a wrong id with a + // business error rather than a "not logged in" one — so without the real + // id a restored session can never be validated and the cache is useless. + UserID int64 `json:"userId" gorm:"not null;default:0"` // CookiesJSON is the cookie jar serialised by sybclient.Client.ExportCookiesJSON. // It is a credential-equivalent secret: never log it, never return it over HTTP. CookiesJSON string `json:"-" gorm:"type:text;not null"` diff --git a/server/app/goauto/sybclient/session.go b/server/app/goauto/sybclient/session.go index e5923ca..9d8d8a4 100644 --- a/server/app/goauto/sybclient/session.go +++ b/server/app/goauto/sybclient/session.go @@ -30,25 +30,41 @@ type SessionStore struct{ db *gorm.DB } func NewSessionStore(db *gorm.DB) *SessionStore { return &SessionStore{db: db} } +// Session is one cached SYB login. +type Session struct { + Username string + // UserID is SYB's numeric account id from the login response. CheckSession + // needs it, so caching cookies without it would make the cache unusable. + UserID int64 + CookiesJSON string + ExpiresAt time.Time +} + // Save replaces the whole session row for username. Replacing rather than // merging is deliberate: a partial update could leave a cookie from a previous // session alongside the new one, and the two jars are not interchangeable. -func (s *SessionStore) Save(ctx context.Context, username, cookiesJSON string, expiresAt time.Time) error { - username = strings.TrimSpace(username) - if username == "" { +func (s *SessionStore) Save(ctx context.Context, session Session) error { + session.Username = strings.TrimSpace(session.Username) + if session.Username == "" { return fmt.Errorf("顺云宝账号不能为空") } - if strings.TrimSpace(cookiesJSON) == "" { + if strings.TrimSpace(session.CookiesJSON) == "" { return fmt.Errorf("顺云宝会话 Cookie 不能为空") } - if expiresAt.IsZero() { + if session.ExpiresAt.IsZero() { return fmt.Errorf("顺云宝会话必须有过期时间") } + if session.UserID <= 0 { + return fmt.Errorf("顺云宝会话必须带上登录返回的 user id，否则恢复后无法校验") + } return s.db.WithContext(ctx).Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "username"}}, - DoUpdates: clause.AssignmentColumns([]string{"cookies_json", "expires_at", "updated_at"}), + DoUpdates: clause.AssignmentColumns([]string{"user_id", "cookies_json", "expires_at", "updated_at"}), }).Create(&models.SYBSession{ - Username: username, CookiesJSON: cookiesJSON, ExpiresAt: expiresAt.UTC(), + Username: session.Username, + UserID: session.UserID, + CookiesJSON: session.CookiesJSON, + ExpiresAt: session.ExpiresAt.UTC(), }).Error } @@ -60,19 +76,22 @@ func (s *SessionStore) Save(ctx context.Context, username, cookiesJSON string, e // expired jar would only produce a confusing "未登录" error one request later. // The row is not deleted here: Load must stay safe to call concurrently, and // the next successful login overwrites it anyway. -func (s *SessionStore) Load(ctx context.Context, username string, now time.Time) (string, time.Time, error) { +func (s *SessionStore) Load(ctx context.Context, username string, now time.Time) (Session, error) { var record models.SYBSession err := s.db.WithContext(ctx).Where("username = ?", strings.TrimSpace(username)).First(&record).Error if errors.Is(err, gorm.ErrRecordNotFound) { - return "", time.Time{}, ErrNoSession + return Session{}, ErrNoSession } if err != nil { - return "", time.Time{}, err + return Session{}, err } if !now.UTC().Before(record.ExpiresAt.UTC()) { - return "", record.ExpiresAt, ErrNoSession + return Session{}, ErrNoSession } - return record.CookiesJSON, record.ExpiresAt, nil + return Session{ + Username: record.Username, UserID: record.UserID, + CookiesJSON: record.CookiesJSON, ExpiresAt: record.ExpiresAt, + }, nil } // Delete drops the cached session for username. Callers use it after the diff --git a/server/app/goauto/sybclient/session_test.go b/server/app/goauto/sybclient/session_test.go index 8ab2062..fb80f2f 100644 --- a/server/app/goauto/sybclient/session_test.go +++ b/server/app/goauto/sybclient/session_test.go @@ -36,18 +36,22 @@ func TestSaveThenLoadRoundTripsCookies(t *testing.T) { now := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) expires := now.Add(24 * time.Hour) - if err := store.Save(ctx, "operator", `[{"name":"SESSION","value":"abc"}]`, expires); err != nil { + if err := store.Save(ctx, Session{Username: "operator", UserID: 4242, CookiesJSON: `[{"name":"SESSION","value":"abc"}]`, ExpiresAt: expires}); err != nil { t.Fatalf("保存会话失败: %v", err) } - cookies, gotExpires, err := store.Load(ctx, "operator", now) + got, err := store.Load(ctx, "operator", now) if err != nil { t.Fatalf("读取会话失败: %v", err) } - if cookies != `[{"name":"SESSION","value":"abc"}]` { - t.Fatalf("Cookie 内容不一致: %q", cookies) + if got.CookiesJSON != `[{"name":"SESSION","value":"abc"}]` { + t.Fatalf("Cookie 内容不一致: %q", got.CookiesJSON) } - if !gotExpires.Equal(expires) { - t.Fatalf("过期时间不一致: 期望 %v，实际 %v", expires, gotExpires) + if !got.ExpiresAt.Equal(expires) { + t.Fatalf("过期时间不一致: 期望 %v，实际 %v", expires, got.ExpiresAt) + } + // 没有 user id 就没法调 /am/user/get 校验会话，缓存等于白存。 + if got.UserID != 4242 { + t.Fatalf("user id 没有被缓存: %d", got.UserID) } } @@ -57,21 +61,21 @@ func TestSaveReplacesExistingSessionForSameAccount(t *testing.T) { ctx := context.Background() now := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) - if err := store.Save(ctx, "operator", `["old"]`, now.Add(time.Hour)); err != nil { + if err := store.Save(ctx, Session{Username: "operator", UserID: 4242, CookiesJSON: `["old"]`, ExpiresAt: now.Add(time.Hour)}); err != nil { t.Fatalf("首次保存失败: %v", err) } - if err := store.Save(ctx, "operator", `["new"]`, now.Add(24*time.Hour)); err != nil { + if err := store.Save(ctx, Session{Username: "operator", UserID: 4242, CookiesJSON: `["new"]`, ExpiresAt: now.Add(24 * time.Hour)}); err != nil { t.Fatalf("覆盖保存失败: %v", err) } - cookies, gotExpires, err := store.Load(ctx, "operator", now) + got, err := store.Load(ctx, "operator", now) if err != nil { t.Fatalf("读取会话失败: %v", err) } - if cookies != `["new"]` { - t.Fatalf("旧 Cookie 没有被覆盖: %q", cookies) + if got.CookiesJSON != `["new"]` { + t.Fatalf("旧 Cookie 没有被覆盖: %q", got.CookiesJSON) } - if !gotExpires.Equal(now.Add(24 * time.Hour)) { - t.Fatalf("过期时间没有被覆盖: %v", gotExpires) + if !got.ExpiresAt.Equal(now.Add(24 * time.Hour)) { + t.Fatalf("过期时间没有被覆盖: %v", got.ExpiresAt) } } @@ -82,10 +86,10 @@ func TestLoadTreatsExpiredSessionAsAbsent(t *testing.T) { ctx := context.Background() now := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) - if err := store.Save(ctx, "operator", `["jar"]`, now.Add(time.Hour)); err != nil { + if err := store.Save(ctx, Session{Username: "operator", UserID: 4242, CookiesJSON: `["jar"]`, ExpiresAt: now.Add(time.Hour)}); err != nil { t.Fatalf("保存会话失败: %v", err) } - if _, _, err := store.Load(ctx, "operator", now.Add(2*time.Hour)); !errors.Is(err, ErrNoSession) { + if _, err := store.Load(ctx, "operator", now.Add(2*time.Hour)); !errors.Is(err, ErrNoSession) { t.Fatalf("过期会话应报 ErrNoSession，实际: %v", err) } } @@ -97,17 +101,17 @@ func TestLoadTreatsSessionExactlyAtExpiryAsAbsent(t *testing.T) { now := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) expires := now.Add(time.Hour) - if err := store.Save(ctx, "operator", `["jar"]`, expires); err != nil { + if err := store.Save(ctx, Session{Username: "operator", UserID: 4242, CookiesJSON: `["jar"]`, ExpiresAt: expires}); err != nil { t.Fatalf("保存会话失败: %v", err) } - if _, _, err := store.Load(ctx, "operator", expires); !errors.Is(err, ErrNoSession) { + if _, err := store.Load(ctx, "operator", expires); !errors.Is(err, ErrNoSession) { t.Fatalf("正好到期的会话应报 ErrNoSession，实际: %v", err) } } func TestLoadReportsErrNoSessionWhenNeverSaved(t *testing.T) { store := NewSessionStore(newSessionTestDB(t)) - if _, _, err := store.Load(context.Background(), "nobody", time.Now()); !errors.Is(err, ErrNoSession) { + if _, err := store.Load(context.Background(), "nobody", time.Now()); !errors.Is(err, ErrNoSession) { t.Fatalf("未保存过的账号应报 ErrNoSession，实际: %v", err) } } @@ -118,17 +122,17 @@ func TestDeleteRemovesOnlyTheNamedAccount(t *testing.T) { now := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) for _, name := range []string{"alice", "bob"} { - if err := store.Save(ctx, name, `["jar"]`, now.Add(time.Hour)); err != nil { + if err := store.Save(ctx, Session{Username: name, UserID: 4242, CookiesJSON: `["jar"]`, ExpiresAt: now.Add(time.Hour)}); err != nil { t.Fatalf("保存 %s 失败: %v", name, err) } } if err := store.Delete(ctx, "alice"); err != nil { t.Fatalf("删除失败: %v", err) } - if _, _, err := store.Load(ctx, "alice", now); !errors.Is(err, ErrNoSession) { + if _, err := store.Load(ctx, "alice", now); !errors.Is(err, ErrNoSession) { t.Fatalf("alice 应已被删除，实际: %v", err) } - if _, _, err := store.Load(ctx, "bob", now); err != nil { + if _, err := store.Load(ctx, "bob", now); err != nil { t.Fatalf("bob 不应受影响，实际: %v", err) } } @@ -140,14 +144,15 @@ func TestSaveRejectsIncompleteInput(t *testing.T) { valid := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) for _, c := range []struct { - name, username, cookies string - expires time.Time + name string + session Session }{ - {"空账号", " ", `["jar"]`, valid}, - {"空Cookie", "operator", " ", valid}, - {"零过期时间", "operator", `["jar"]`, time.Time{}}, + {"空账号", Session{Username: " ", UserID: 1, CookiesJSON: `["jar"]`, ExpiresAt: valid}}, + {"空Cookie", Session{Username: "operator", UserID: 1, CookiesJSON: " ", ExpiresAt: valid}}, + {"零过期时间", Session{Username: "operator", UserID: 1, CookiesJSON: `["jar"]`}}, + {"缺userID", Session{Username: "operator", CookiesJSON: `["jar"]`, ExpiresAt: valid}}, } { - if err := store.Save(ctx, c.username, c.cookies, c.expires); err == nil { + if err := store.Save(ctx, c.session); err == nil { t.Fatalf("%s 应该被拒绝", c.name) } } diff --git a/server/app/goauto/sybimport/import_handler.go b/server/app/goauto/sybimport/import_handler.go new file mode 100644 index 0000000..cd2bf5d --- /dev/null +++ b/server/app/goauto/sybimport/import_handler.go @@ -0,0 +1,111 @@ +package sybimport + +import ( + "context" + "net/http" + "sync" + "time" + + "go-admin/app/goauto/sybclient" + "go-admin/config" + + "github.com/gin-gonic/gin" +) + +// importTimeout bounds one import run. A month-wide range over a busy account +// is thousands of remote reads, so the ceiling is generous — but it must exist, +// or a stalled SYB response would pin the request forever. +const importTimeout = 30 * time.Minute + +// importGate makes imports mutually exclusive. Two concurrent runs over +// overlapping dates would race on the same (order_code, detail_id) rows: the +// writes are idempotent so the data would survive, but the two reports would +// each undercount, and the operator would have no way to tell which is right. +var importGate = struct { + sync.Mutex + running bool +}{} + +type ImportRequest struct { + DateFrom string `json:"dateFrom"` + DateTo string `json:"dateTo"` +} + +// Import pulls shipment orders from SYB for a date range and folds them into +// the archive. +// +// `[必须]` This is the only endpoint that reaches out to SYB. It performs reads +// only — no SYB write endpoint is called from anywhere in GoAuto. +func (handler Handler) Import(c *gin.Context) { + var request ImportRequest + if err := c.ShouldBindJSON(&request); err != nil { + writeError(c, invalidRequest("请求体必须是合法 JSON，且包含 dateFrom 和 dateTo")) + return + } + if request.DateFrom == "" || request.DateTo == "" { + writeError(c, invalidRequest("dateFrom 和 dateTo 不能为空，格式为 YYYY-MM-DD")) + return + } + if _, err := splitDateRange(request.DateFrom, request.DateTo); err != nil { + writeError(c, invalidRequest(err.Error())) + return + } + + settings := config.ExtConfig.SYB.Resolved() + if !settings.HasCredentials() { + writeError(c, invalidRequest( + "顺云宝账号未配置：请设置环境变量 GOAUTO_SYB_USERNAME 和 GOAUTO_SYB_PASSWORD 后重启服务端")) + return + } + + service, ok := handler.service(c) + if !ok { + return + } + + importGate.Lock() + if importGate.running { + importGate.Unlock() + writeError(c, &ServiceError{Code: CodeInvalidRequest, Message: "已有一个导入任务正在执行，请等它结束后再试"}) + return + } + importGate.running = true + importGate.Unlock() + defer func() { + importGate.Lock() + importGate.running = false + importGate.Unlock() + }() + + // Detached from the request context on purpose: an operator closing the tab + // must not abort a half-finished import, which would leave a partial range + // imported with nobody holding the report that says how far it got. + ctx, cancel := context.WithTimeout(context.Background(), importTimeout) + defer cancel() + + client, err := Connect(ctx, sybclient.NewSessionStore(service.DB), ConnectConfig{ + BaseURL: settings.BaseURL, + Username: settings.Username, + Password: settings.Password, + OcrURL: settings.OcrURL, + OcrMaxAttempts: settings.OcrMaxAttempts, + }) + if err != nil { + writeError(c, &ServiceError{Code: CodeInvalidRequest, Message: err.Error()}) + return + } + + report, err := Sync(ctx, service.DB, client, SyncConfig{ + PageSize: settings.PageSize, MaxMatches: settings.MaxMatches, + }, request.DateFrom, request.DateTo) + if err != nil { + // The partial report goes back with the error: rows already written are + // kept (a re-run overwrites them), so the operator needs to see how far + // it got, not just that it failed. + c.JSON(http.StatusBadGateway, gin.H{ + "code": CodeInvalidRequest, "message": err.Error(), "data": report, + }) + return + } + c.JSON(http.StatusOK, gin.H{"code": 200, "data": report}) +} diff --git a/server/app/goauto/sybimport/router.go b/server/app/goauto/sybimport/router.go index 32a3832..64422fc 100644 --- a/server/app/goauto/sybimport/router.go +++ b/server/app/goauto/sybimport/router.go @@ -7,11 +7,9 @@ import ( jwt "github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth" ) -// InitRouter mounts the read/reparse/manual-correct surface only. There is no -// create/import endpoint: no SYB credentials or fetch client exist in this -// repo yet (see package doc), so rows only appear here once something else -// calls sybimport.ApplyDetail directly (currently only from tests). Wiring a -// live import trigger is a separate, explicitly deferred piece of work. +// InitRouter mounts the read/reparse/manual-correct surface plus the import +// trigger (#48). Import is the only route that reaches out to SYB, and it only +// reads: no SYB write endpoint is called from anywhere in GoAuto. func InitRouter(engine *gin.Engine, auth *jwt.GinJWTMiddleware) { handler := Handler{} admin := engine.Group("/api/admin/v1/syb-products").Use(auth.MiddlewareFunc()).Use(middleware.AuthCheckRole()) @@ -19,5 +17,6 @@ func InitRouter(engine *gin.Engine, auth *jwt.GinJWTMiddleware) { admin.GET("/:productId", handler.Detail) admin.POST("/:productId/reparse", handler.Reparse) admin.POST("/reparse-batch", handler.ReparseBatch) + admin.POST("/import", handler.Import) admin.PATCH("/:productId/correction", handler.ManualCorrect) } diff --git a/server/app/goauto/sybimport/router_test.go b/server/app/goauto/sybimport/router_test.go new file mode 100644 index 0000000..837c4af --- /dev/null +++ b/server/app/goauto/sybimport/router_test.go @@ -0,0 +1,37 @@ +package sybimport + +import ( + "testing" + + "github.com/gin-gonic/gin" +) + +// gin 在注册阶段就会因为路由冲突 panic，编译期发现不了。 +// /import 和 /:productId 处在同一层，必须确认两者能共存。 +func TestRoutesRegisterWithoutConflict(t *testing.T) { + gin.SetMode(gin.TestMode) + engine := gin.New() + handler := Handler{} + group := engine.Group("/api/admin/v1/syb-products") + group.GET("", handler.List) + group.GET("/:productId", handler.Detail) + group.POST("/:productId/reparse", handler.Reparse) + group.POST("/reparse-batch", handler.ReparseBatch) + group.POST("/import", handler.Import) + + want := map[string]bool{ + "GET /api/admin/v1/syb-products": false, + "GET /api/admin/v1/syb-products/:productId": false, + "POST /api/admin/v1/syb-products/:productId/reparse": false, + "POST /api/admin/v1/syb-products/reparse-batch": false, + "POST /api/admin/v1/syb-products/import": false, + } + for _, route := range engine.Routes() { + want[route.Method+" "+route.Path] = true + } + for route, registered := range want { + if !registered { + t.Fatalf("路由未注册: %s", route) + } + } +} diff --git a/server/app/goauto/sybimport/sync.go b/server/app/goauto/sybimport/sync.go new file mode 100644 index 0000000..29498d4 --- /dev/null +++ b/server/app/goauto/sybimport/sync.go @@ -0,0 +1,366 @@ +package sybimport + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "go-admin/app/goauto/sybclient" + + "gorm.io/gorm" +) + +// SyncConfig is the subset of the SYB settings the orchestration needs. It is +// a plain struct rather than config.SYB so this package does not depend on the +// server's configuration wiring, which keeps it testable without settings.yml. +type SyncConfig struct { + PageSize int + // MaxMatches caps how many shipment orders one sync may touch. Real data + // makes this matter: a single week held 7404 orders, so a careless range + // blows past any sane budget (docs/12-syb-erp-interface.md §8). + MaxMatches int +} + +const ( + // detailBatch is SYB's hard limit on listByStock (§6). + detailBatch = 100 + // maxSyncDays bounds one request's window. It is a guard against a typo in + // the date range turning into tens of thousands of remote reads before + // MaxMatches trips. + maxSyncDays = 31 +) + +// SyncReport summarises one sync run. +type SyncReport struct { + From string `json:"from"` + To string `json:"to"` + OrderCount int `json:"orderCount"` + DetailCount int `json:"detailCount"` + Created int `json:"created"` + Updated int `json:"updated"` + StartedAt time.Time `json:"startedAt"` + FinishedAt time.Time `json:"finishedAt"` +} + +// Sync pulls every shipment order in [dateFrom, dateTo] and folds each detail +// line into the SYB/Shopee archive through ApplyDetail. +// +// `[必须]` The caller supplies an already-logged-in client. Session handling is +// a separate concern (see Connect); a session that dies mid-run surfaces as +// sybclient.ErrSessionInvalid and is treated like any other mid-run failure. +// +// `[必须]` Failure stops the run immediately. Rows already written are NOT +// rolled back — ApplyDetail is idempotent on (order_code, detail_id), so a +// re-run overwrites them rather than duplicating. What must not happen is +// reporting a partial run as a complete one, which would let the missing +// orders go unnoticed forever. +// +// `[必须]` Every day is verified for completeness before anything is written: +// the per-day total is re-read after paging and must not have drifted. SYB's +// list endpoint returns the *page* size in `total` (§4.3, confirmed against +// live data), so the paging loop is driven by listTotal, never by list.total. +func Sync(ctx context.Context, db *gorm.DB, client *sybclient.Client, cfg SyncConfig, dateFrom, dateTo string) (SyncReport, error) { + report := SyncReport{From: dateFrom, To: dateTo, StartedAt: time.Now().UTC()} + + dates, err := splitDateRange(dateFrom, dateTo) + if err != nil { + return report, err + } + pageSize := cfg.PageSize + if pageSize <= 0 { + pageSize = 20 + } + maxMatches := cfg.MaxMatches + if maxMatches <= 0 { + maxMatches = 10000 + } + + // Pre-flight: total up the whole range before fetching anything, so an + // over-budget range fails without having written a single row. + type dayPlan struct { + date string + total int + } + plans := make([]dayPlan, 0, len(dates)) + rangeTotal := 0 + for _, date := range dates { + total, err := client.ListTotal(ctx, date, date, pageSize) + if err != nil { + return report, fmt.Errorf("查询 %s 货运单总数失败: %w", date, err) + } + if total < 0 { + return report, fmt.Errorf("查询 %s 货运单总数返回负数 %d", date, total) + } + rangeTotal += total + if rangeTotal > maxMatches { + return report, fmt.Errorf( + "日期范围 %s ~ %s 的货运单总数已超过单次同步上限 %d（截至 %s 已 %d 张），"+ + "请缩小日期范围；实测最近 7 天可达 7000 张以上", + dateFrom, dateTo, maxMatches, date, rangeTotal) + } + plans = append(plans, dayPlan{date: date, total: total}) + } + + for _, plan := range plans { + if plan.total == 0 { + continue + } + rows, err := loadDailyList(ctx, client, plan.date, pageSize, plan.total) + if err != nil { + return report, err + } + report.OrderCount += len(rows) + + byID := make(map[int64]sybclient.StockRow, len(rows)) + ids := make([]int64, 0, len(rows)) + for _, row := range rows { + byID[row.ID] = row + ids = append(ids, row.ID) + } + + for start := 0; start < len(ids); start += detailBatch { + end := start + detailBatch + if end > len(ids) { + end = len(ids) + } + batch := ids[start:end] + details, err := client.DetailListByStock(ctx, batch) + if err != nil { + return report, fmt.Errorf("拉取 %s 货运单明细失败（本次同步停止；"+ + "已写入的数据保留，重跑会按 (order_code, detail_id) 覆盖）: %w", plan.date, err) + } + if err := validateDetailBatch(batch, details); err != nil { + return report, fmt.Errorf("%s 货运单明细不完整：%w；本次同步停止", plan.date, err) + } + for _, detail := range details { + if err := applyStockDetail(ctx, db, byID[detail.ID], detail, &report); err != nil { + return report, err + } + } + } + } + + report.FinishedAt = time.Now().UTC() + return report, nil +} + +// loadDailyList pages through one day and refuses to return a list it cannot +// prove is complete. +// +// `[必须]` The loop bound comes from expectedTotal (listTotal), because +// list.total is the current page's row count, not the filtered total (§4.3). +// Driving the loop with the response's own total would stop after page one. +func loadDailyList(ctx context.Context, client *sybclient.Client, date string, pageSize, expectedTotal int) ([]sybclient.StockRow, error) { + rows := make([]sybclient.StockRow, 0, expectedTotal) + seen := make(map[int64]struct{}, expectedTotal) + + for start := 0; start < expectedTotal; start += pageSize { + pageIndex := start/pageSize + 1 + page, pageCount, err := client.ListPage(ctx, date, date, start, pageIndex, pageSize) + if err != nil { + return nil, fmt.Errorf("拉取 %s 货运单列表第 %d 页失败（已获取 %d/%d 张）: %w", + date, pageIndex, len(rows), expectedTotal, err) + } + expectedPageCount := pageSize + if remaining := expectedTotal - start; remaining < pageSize { + expectedPageCount = remaining + } + if pageCount != expectedPageCount || len(page) != expectedPageCount { + return nil, fmt.Errorf("%s 货运单列表第 %d 页不完整：预期 %d 行，实际 %d 行；"+ + "分页期间数据发生变化，本次同步停止", date, pageIndex, expectedPageCount, len(page)) + } + for _, row := range page { + if _, duplicate := seen[row.ID]; duplicate { + continue + } + seen[row.ID] = struct{}{} + rows = append(rows, row) + } + } + + // Re-read the total: if it moved while we paged, some order was inserted or + // removed underneath us and the snapshot we hold has a hole in it. + afterTotal, err := client.ListTotal(ctx, date, date, pageSize) + if err != nil { + return nil, fmt.Errorf("分页后重新查询 %s 货运单总数失败: %w", date, err) + } + if afterTotal != expectedTotal { + return nil, fmt.Errorf("%s 货运单总数在分页期间从 %d 变为 %d，本次同步停止", + date, expectedTotal, afterTotal) + } + if len(rows) != expectedTotal { + return nil, fmt.Errorf("%s 货运单列表不完整：预期 %d 张，分页后只有 %d 个唯一 ID", + date, expectedTotal, len(rows)) + } + return rows, nil +} + +// validateDetailBatch rejects a detail response that does not exactly answer +// what was asked. Silently accepting a short response would drop shipment +// orders from the import while still reporting success. +func validateDetailBatch(requested []int64, details []sybclient.StockDetail) error { + wanted := make(map[int64]struct{}, len(requested)) + for _, id := range requested { + wanted[id] = struct{}{} + } + seen := make(map[int64]struct{}, len(details)) + for _, detail := range details { + if _, ok := wanted[detail.ID]; !ok { + return fmt.Errorf("响应包含未请求的货运单 id=%d", detail.ID) + } + if _, duplicate := seen[detail.ID]; duplicate { + return fmt.Errorf("响应重复返回货运单 id=%d", detail.ID) + } + seen[detail.ID] = struct{}{} + } + for _, id := range requested { + if _, ok := seen[id]; !ok { + return fmt.Errorf("响应缺少货运单 id=%d", id) + } + } + return nil +} + +// applyStockDetail writes one shipment order's items. Each item is its own +// ApplyDetail call and its own transaction — a several-thousand-row sync must +// not sit in one long-held transaction. +func applyStockDetail(ctx context.Context, db *gorm.DB, row sybclient.StockRow, detail sybclient.StockDetail, report *SyncReport) error { + order := OrderInput{ + Code: detail.Code, + StockID: uint64(detail.ID), + ShopName: stringField(row.Raw, "shopName"), + } + if order.Code == "" { + order.Code = row.Code + } + for _, item := range detail.Details { + raw, err := json.Marshal(item.Raw) + if err != nil { + return fmt.Errorf("货运单 %s 明细 %d 原始数据编码失败: %w", order.Code, item.ID, err) + } + result, err := ApplyDetail(ctx, db, order, DetailInput{ + ID: uint64(item.ID), + ProductID: uint64(item.ProductID), + ProductQty: int64(item.ProductQty), + ProductPrice: item.ProductPrice, + ProductSpec: item.ProductSpec, + ProductTitle: item.ProductTitle, + ProductThumb: uint64(item.ProductThumb), + Raw: raw, + }) + if err != nil { + return fmt.Errorf("写入货运单 %s 明细 %d 失败（本次同步停止；已写入的数据保留）: %w", + order.Code, item.ID, err) + } + report.DetailCount++ + switch result.Outcome { + case OutcomeCreated: + report.Created++ + case OutcomeUpdated: + report.Updated++ + } + } + return nil +} + +func stringField(raw map[string]any, key string) string { + if raw == nil { + return "" + } + value, _ := raw[key].(string) + return value +} + +// splitDateRange expands an inclusive YYYY-MM-DD range into single days, +// because SYB's totals and paging are only self-consistent within one day. +func splitDateRange(dateFrom, dateTo string) ([]string, error) { + from, err := time.Parse("2006-01-02", dateFrom) + if err != nil { + return nil, fmt.Errorf("起始日期格式应为 YYYY-MM-DD: %q", dateFrom) + } + to, err := time.Parse("2006-01-02", dateTo) + if err != nil { + return nil, fmt.Errorf("结束日期格式应为 YYYY-MM-DD: %q", dateTo) + } + if to.Before(from) { + return nil, fmt.Errorf("结束日期 %s 不能早于起始日期 %s", dateTo, dateFrom) + } + dates := make([]string, 0, 8) + for day := from; !day.After(to); day = day.AddDate(0, 0, 1) { + dates = append(dates, day.Format("2006-01-02")) + if len(dates) > maxSyncDays { + return nil, fmt.Errorf("单次同步最多 %d 天，请缩小日期范围", maxSyncDays) + } + } + return dates, nil +} + +// Connect returns a logged-in client, reusing the cached session when one is +// still valid so a routine sync does not burn a captcha round-trip. +// +// `[必须]` A cached session is only discarded when SYB explicitly says it is +// invalid. A timeout or a 5xx leaves it in place (§3.5): treating a network +// blip as a logout would trigger needless logins and could throw away a +// perfectly good session. +func Connect(ctx context.Context, store *sybclient.SessionStore, cfg ConnectConfig) (*sybclient.Client, error) { + if cfg.Username == "" || cfg.Password == "" { + return nil, errors.New("顺云宝账号或密码未配置，请设置 GOAUTO_SYB_USERNAME 和 GOAUTO_SYB_PASSWORD") + } + client, err := sybclient.New(cfg.BaseURL) + if err != nil { + return nil, err + } + + cached, err := store.Load(ctx, cfg.Username, time.Now()) + switch { + case err == nil: + if importErr := client.ImportCookiesJSON(cached.CookiesJSON); importErr == nil { + if checkErr := client.CheckSession(ctx, cached.UserID, cfg.Username); checkErr == nil { + return client, nil + } else if errors.Is(checkErr, sybclient.ErrSessionInvalid) { + if delErr := store.Delete(ctx, cfg.Username); delErr != nil { + return nil, delErr + } + } + } + case errors.Is(err, sybclient.ErrNoSession): + // Nothing cached; fall through to a fresh login. + default: + return nil, err + } + + if cfg.OcrURL == "" { + return nil, errors.New("顺云宝会话已失效，且未配置验证码识别服务；请配置 extend.syb.ocrurl 或改用手工登录") + } + ocr, err := sybclient.NewOcrClient(cfg.OcrURL, 0) + if err != nil { + return nil, err + } + result, reason := client.LoginWithOCR(ctx, ocr, cfg.Username, cfg.Password, cfg.OcrMaxAttempts) + if result == nil { + return nil, fmt.Errorf("顺云宝自动登录失败，需要手工输入验证码: %s", reason) + } + jar, err := client.ExportCookiesJSON() + if err != nil { + return nil, err + } + if err := store.Save(ctx, sybclient.Session{ + Username: cfg.Username, UserID: result.User.ID, + CookiesJSON: jar, ExpiresAt: result.ExpiresAt, + }); err != nil { + return nil, err + } + return client, nil +} + +// ConnectConfig carries the login settings. The password is passed through and +// never persisted: only cookies are cached. +type ConnectConfig struct { + BaseURL string + Username string + Password string + OcrURL string + OcrMaxAttempts int +} diff --git a/server/app/goauto/sybimport/sync_test.go b/server/app/goauto/sybimport/sync_test.go new file mode 100644 index 0000000..58fcf28 --- /dev/null +++ b/server/app/goauto/sybimport/sync_test.go @@ -0,0 +1,277 @@ +package sybimport + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "go-admin/app/goauto/migrations" + "go-admin/app/goauto/sybclient" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func newSyncTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared&_foreign_keys=on", + strings.ReplaceAll(t.Name(), "/", "_"))), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatalf("打开测试数据库失败: %v", err) + } + if err := migrations.Migrate(db); err != nil { + t.Fatalf("迁移失败: %v", err) + } + return db +} + +// fakeSYB is a stand-in for the SYB list/detail endpoints, deliberately +// reproducing the quirk confirmed against live data: /am/stock/list returns the +// CURRENT PAGE's row count in `total`, while /am/stock/listTotal returns the +// filtered total (§4.3). +type fakeSYB struct { + perDay map[string]int // date -> order count + // totalAfterPaging, when non-nil, overrides listTotal from the Nth call on, + // simulating orders appearing or disappearing mid-sync. + listTotalCalls int + totalOverride map[int]int + shortPageAtIndex int + detailDropID int64 +} + +func (f *fakeSYB) server(t *testing.T) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/am/stock/listTotal", func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + date := f.dateOf(body) + f.listTotalCalls++ + if override, ok := f.totalOverride[f.listTotalCalls]; ok { + writeEnvelope(w, override) + return + } + writeEnvelope(w, f.perDay[date]) + }) + mux.HandleFunc("/am/stock/list", func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + date := f.dateOf(body) + start := int(body["start"].(float64)) + pageSize := int(body["length"].(float64)) + total := f.perDay[date] + + rows := []map[string]any{} + for i := start; i < total && len(rows) < pageSize; i++ { + rows = append(rows, map[string]any{ + "id": float64(1000 + i), + "code": fmt.Sprintf("ORD%s%03d", strings.ReplaceAll(date, "-", ""), i), + "shopName": "测试店铺", + }) + } + if f.shortPageAtIndex > 0 && start/pageSize+1 == f.shortPageAtIndex && len(rows) > 0 { + rows = rows[:len(rows)-1] + } + writeEnvelope(w, map[string]any{"list": rows, "total": len(rows)}) + }) + mux.HandleFunc("/am/stock/detail/listByStock", func(w http.ResponseWriter, r *http.Request) { + var body struct { + IDs []int64 `json:"ids"` + } + json.NewDecoder(r.Body).Decode(&body) + list := []map[string]any{} + for _, id := range body.IDs { + if id == f.detailDropID { + continue + } + list = append(list, map[string]any{ + "id": id, + "code": fmt.Sprintf("ORD-%d", id), + "details": []any{map[string]any{ + "id": float64(id*10 + 1), "productId": float64(9001), + "productTitle": "测试商品", "productSpec": "白色,L", + "productQty": float64(2), "productPrice": 39.5, "productThumb": float64(77), + }}, + }) + } + writeEnvelope(w, map[string]any{"list": list}) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +func (f *fakeSYB) dateOf(body map[string]any) string { + queries, _ := body["queries"].([]any) + for _, q := range queries { + m, _ := q.(map[string]any) + if value, ok := m["dvalue"].(string); ok && strings.Contains(value, ",") { + return strings.Split(value, ",")[0] + } + } + return "" +} + +func writeEnvelope(w http.ResponseWriter, data any) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{"status": true, "msg": "ok", "data": data}) +} + +func newSyncClient(t *testing.T, f *fakeSYB) *sybclient.Client { + t.Helper() + client, err := sybclient.New(f.server(t).URL) + if err != nil { + t.Fatalf("创建客户端失败: %v", err) + } + return client +} + +// 分页必须由 listTotal 驱动。若误用 list 响应里的 total 当筛选总数， +// 循环会在第一页就停下——这是 §4.3 最容易踩的坑。 +func TestSyncPagesBeyondTheFirstPage(t *testing.T) { + db := newSyncTestDB(t) + f := &fakeSYB{perDay: map[string]int{"2026-08-01": 25}} + report, err := Sync(context.Background(), db, newSyncClient(t, f), + SyncConfig{PageSize: 10, MaxMatches: 1000}, "2026-08-01", "2026-08-01") + if err != nil { + t.Fatalf("同步失败: %v", err) + } + if report.OrderCount != 25 { + t.Fatalf("应拉到 25 张货运单（3 页），实际 %d——分页很可能停在第一页", report.OrderCount) + } + if report.DetailCount != 25 { + t.Fatalf("应写入 25 条明细，实际 %d", report.DetailCount) + } + if report.Created != 25 { + t.Fatalf("首次同步应全部是新增，实际 %d", report.Created) + } +} + +// 重跑同一范围必须幂等：按 (order_code, detail_id) 覆盖，不产生重复行。 +func TestSyncIsIdempotentAcrossRuns(t *testing.T) { + db := newSyncTestDB(t) + f := &fakeSYB{perDay: map[string]int{"2026-08-01": 5}} + cfg := SyncConfig{PageSize: 10, MaxMatches: 1000} + + if _, err := Sync(context.Background(), db, newSyncClient(t, f), cfg, "2026-08-01", "2026-08-01"); err != nil { + t.Fatalf("首次同步失败: %v", err) + } + second, err := Sync(context.Background(), db, newSyncClient(t, f), cfg, "2026-08-01", "2026-08-01") + if err != nil { + t.Fatalf("重跑失败: %v", err) + } + if second.Created != 0 || second.Updated != 5 { + t.Fatalf("重跑应全部是更新，实际 created=%d updated=%d", second.Created, second.Updated) + } + var count int64 + db.Table("syb_product").Count(&count) + if count != 5 { + t.Fatalf("重跑不应产生重复行，实际 %d 行", count) + } +} + +// 超过上限必须在**写库之前**失败，否则会留下一半数据还报错。 +func TestSyncRefusesOverBudgetRangeBeforeWritingAnything(t *testing.T) { + db := newSyncTestDB(t) + f := &fakeSYB{perDay: map[string]int{"2026-08-01": 40, "2026-08-02": 40}} + _, err := Sync(context.Background(), db, newSyncClient(t, f), + SyncConfig{PageSize: 10, MaxMatches: 50}, "2026-08-01", "2026-08-02") + if err == nil { + t.Fatal("超过上限应报错") + } + if !strings.Contains(err.Error(), "上限") { + t.Fatalf("错误信息应说明超过上限: %v", err) + } + var count int64 + db.Table("syb_product").Count(&count) + if count != 0 { + t.Fatalf("超限时不应写入任何数据，实际 %d 行", count) + } +} + +// 分页期间总数变化说明快照有洞，必须停下，不能报告成功。 +func TestSyncStopsWhenTotalDriftsDuringPaging(t *testing.T) { + db := newSyncTestDB(t) + f := &fakeSYB{ + perDay: map[string]int{"2026-08-01": 10}, + // 第 1 次是预检，第 2 次是分页后的复核——让它返回不同的数。 + totalOverride: map[int]int{2: 11}, + } + _, err := Sync(context.Background(), db, newSyncClient(t, f), + SyncConfig{PageSize: 10, MaxMatches: 1000}, "2026-08-01", "2026-08-01") + if err == nil { + t.Fatal("总数漂移应报错，不能当成同步成功") + } + if !strings.Contains(err.Error(), "变为") { + t.Fatalf("错误信息应说明总数发生变化: %v", err) + } +} + +// 某页行数少于预期同样是不完整快照。 +func TestSyncStopsOnShortPage(t *testing.T) { + db := newSyncTestDB(t) + f := &fakeSYB{perDay: map[string]int{"2026-08-01": 25}, shortPageAtIndex: 2} + _, err := Sync(context.Background(), db, newSyncClient(t, f), + SyncConfig{PageSize: 10, MaxMatches: 1000}, "2026-08-01", "2026-08-01") + if err == nil { + t.Fatal("缺行的分页应报错") + } + if !strings.Contains(err.Error(), "不完整") { + t.Fatalf("错误信息应说明列表不完整: %v", err) + } +} + +// 明细响应少返回一张货运单时必须报错。静默接受会让这张单永远漏掉， +// 而同步却报告成功。 +func TestSyncStopsWhenDetailResponseIsMissingAnOrder(t *testing.T) { + db := newSyncTestDB(t) + f := &fakeSYB{perDay: map[string]int{"2026-08-01": 3}, detailDropID: 1001} + _, err := Sync(context.Background(), db, newSyncClient(t, f), + SyncConfig{PageSize: 10, MaxMatches: 1000}, "2026-08-01", "2026-08-01") + if err == nil { + t.Fatal("明细缺货运单应报错") + } + if !strings.Contains(err.Error(), "缺少货运单") { + t.Fatalf("错误信息应指出缺了哪张货运单: %v", err) + } +} + +func TestSyncSpansMultipleDays(t *testing.T) { + db := newSyncTestDB(t) + f := &fakeSYB{perDay: map[string]int{"2026-08-01": 3, "2026-08-02": 0, "2026-08-03": 2}} + report, err := Sync(context.Background(), db, newSyncClient(t, f), + SyncConfig{PageSize: 10, MaxMatches: 1000}, "2026-08-01", "2026-08-03") + if err != nil { + t.Fatalf("同步失败: %v", err) + } + if report.OrderCount != 5 { + t.Fatalf("三天合计应为 5 张，实际 %d", report.OrderCount) + } +} + +func TestSplitDateRangeRejectsBadInput(t *testing.T) { + for _, c := range []struct{ name, from, to string }{ + {"格式错误", "2026/08/01", "2026-08-02"}, + {"结束早于开始", "2026-08-05", "2026-08-01"}, + {"跨度过大", "2026-01-01", "2026-06-01"}, + } { + if _, err := splitDateRange(c.from, c.to); err == nil { + t.Fatalf("%s 应该被拒绝", c.name) + } + } +} + +func TestSplitDateRangeIsInclusive(t *testing.T) { + dates, err := splitDateRange("2026-08-01", "2026-08-03") + if err != nil { + t.Fatalf("拆分失败: %v", err) + } + if len(dates) != 3 || dates[0] != "2026-08-01" || dates[2] != "2026-08-03" { + t.Fatalf("首尾日期都应包含在内: %v", dates) + } +} diff --git a/web/src/api/goauto/syb-products.js b/web/src/api/goauto/syb-products.js index 2086cbc..396de69 100644 --- a/web/src/api/goauto/syb-products.js +++ b/web/src/api/goauto/syb-products.js @@ -19,3 +19,9 @@ export function reparseSybProductsBatch(data) { export function correctSybProduct(productId, data) { return request({ url: `/api/admin/v1/syb-products/${productId}/correction`, method: 'patch', data }) } + +// 从 SYB 拉取指定日期范围的货运单并入库。耗时可能很长（一周可达数千张）， +// 所以单独放大超时，不用 request 的默认值。 +export function importSybProducts(data) { + return request({ url: '/api/admin/v1/syb-products/import', method: 'post', data, timeout: 30 * 60 * 1000 }) +} diff --git a/web/src/views/goauto/syb-products/index.vue b/web/src/views/goauto/syb-products/index.vue index 47da737..00841df 100644 --- a/web/src/views/goauto/syb-products/index.vue +++ b/web/src/views/goauto/syb-products/index.vue @@ -2,8 +2,8 @@