feat(goauto): add SYB product variation filters (#269)

This commit is contained in:
QiuSW
2026-09-11 16:20:26 +08:00
parent 79929036bb
commit 652bf30f5a
18 changed files with 593 additions and 59 deletions
+2
View File
@@ -18,6 +18,7 @@ import (
goautoshopeeproduct "go-admin/app/goauto/shopeeproduct"
goautosybimport "go-admin/app/goauto/sybimport"
goautosybinnercode "go-admin/app/goauto/sybinnercode"
goautosybproductfilter "go-admin/app/goauto/sybproductfilter"
goautosybshop "go-admin/app/goauto/sybshop"
goautotask "go-admin/app/goauto/task"
common "go-admin/common/middleware"
@@ -67,4 +68,5 @@ func InitRouter() {
goautosybimport.InitRouter(r, authMiddleware)
goautosybinnercode.InitRouter(r, authMiddleware)
goautosybshop.InitRouter(r, authMiddleware)
goautosybproductfilter.InitRouter(r, authMiddleware)
}
+24 -19
View File
@@ -3,18 +3,19 @@ package access
import "strings"
const (
ModuleDevices = "devices"
ModulePDDProducts = "pdd_products"
ModuleShopeeProducts = "shopee_products"
ModuleSYBProducts = "syb_products"
ModuleSYBShops = "syb_shops"
ModuleSYBSyncRuns = "syb_sync_runs"
ModuleSYBInnerCodes = "syb_inner_codes"
ModuleCollectionRules = "collection_rules"
ModulePurchaseRules = "purchase_rules"
ModuleCollectionTasks = "collection_tasks"
ModulePurchaseTasks = "purchase_tasks"
ModuleAIMatching = "ai_matching"
ModuleDevices = "devices"
ModulePDDProducts = "pdd_products"
ModuleShopeeProducts = "shopee_products"
ModuleSYBProducts = "syb_products"
ModuleSYBShops = "syb_shops"
ModuleSYBProductFilters = "syb_product_filters"
ModuleSYBSyncRuns = "syb_sync_runs"
ModuleSYBInnerCodes = "syb_inner_codes"
ModuleCollectionRules = "collection_rules"
ModulePurchaseRules = "purchase_rules"
ModuleCollectionTasks = "collection_tasks"
ModulePurchaseTasks = "purchase_tasks"
ModuleAIMatching = "ai_matching"
)
// ModuleDefinition is the single source of truth shared by menu migration,
@@ -77,6 +78,7 @@ var goAutoMenuGroupMetadata = []MenuGroupDefinition{
Sort: 51,
ModuleKeys: []string{
ModuleSYBShops,
ModuleSYBProductFilters,
ModuleCollectionRules,
ModulePurchaseRules,
ModuleDevices,
@@ -91,13 +93,14 @@ var goAutoModuleMetadata = []ModuleDefinition{
{Key: ModuleShopeeProducts, Title: "虾皮商品", Path: "/shopee-products", RouteName: "GoAutoShopeeProducts", Component: "/goauto/shopee-products/index", Icon: "goods", Sort: 52, PurchaserDefault: true},
{Key: ModuleSYBProducts, Title: "SYB 商品", Path: "/syb-products", RouteName: "GoAutoSybProducts", Component: "/goauto/syb-products/index", Icon: "tickets", Sort: 53, PurchaserDefault: true},
{Key: ModuleSYBShops, Title: "SYB 店铺", Path: "/syb-shops", RouteName: "GoAutoSybShops", Component: "/goauto/syb-shops/index", Icon: "shopping", Sort: 54, PurchaserDefault: true},
{Key: ModuleSYBSyncRuns, Title: "SYB 同步记录", Path: "/syb-sync-runs", RouteName: "GoAutoSybSyncRuns", Component: "/goauto/syb-sync-runs/index", Icon: "time", Sort: 55, PurchaserDefault: true},
{Key: ModuleSYBInnerCodes, Title: "档口入库码", Path: "/syb-inner-codes", RouteName: "GoAutoSybInnerCodes", Component: "/goauto/syb-inner-codes/index", Icon: "list", Sort: 56, PurchaserDefault: true},
{Key: ModuleCollectionRules, Title: "采集规则", Path: "/collection-rules", RouteName: "GoAutoCollectionRules", Component: "/goauto/collection-rules/index", Icon: "code", Sort: 57, PurchaserDefault: true},
{Key: ModulePurchaseRules, Title: "采购规则", Path: "/purchase-rules", RouteName: "GoAutoPurchaseRules", Component: "/goauto/purchase-rules/index", Icon: "setting", Sort: 58, PurchaserHardHidden: true},
{Key: ModuleCollectionTasks, Title: "采集任务", Path: "/collection-tasks", RouteName: "GoAutoCollectionTasks", Component: "/goauto/collection-tasks/index", Icon: "list", Sort: 59, PurchaserDefault: true},
{Key: ModulePurchaseTasks, Title: "采购管理", Path: "/purchase-tasks", RouteName: "GoAutoPurchaseTasks", Component: "/goauto/purchase-tasks/index", Icon: "shopping", Sort: 60, PurchaserDefault: true},
{Key: ModuleAIMatching, Title: "AI 规格匹配", Path: "/ai-matching-settings", RouteName: "GoAutoAiMatchingSettings", Component: "/goauto/ai-matching-settings/index", Icon: "setting", Sort: 61, PurchaserHardHidden: true},
{Key: ModuleSYBProductFilters, Title: "SYB 商品过滤", Path: "/syb-product-filters", RouteName: "GoAutoSybProductFilters", Component: "/goauto/syb-product-filters/index", Icon: "filter", Sort: 55, PurchaserDefault: true},
{Key: ModuleSYBSyncRuns, Title: "SYB 同步记录", Path: "/syb-sync-runs", RouteName: "GoAutoSybSyncRuns", Component: "/goauto/syb-sync-runs/index", Icon: "time", Sort: 56, PurchaserDefault: true},
{Key: ModuleSYBInnerCodes, Title: "档口入库码", Path: "/syb-inner-codes", RouteName: "GoAutoSybInnerCodes", Component: "/goauto/syb-inner-codes/index", Icon: "list", Sort: 57, PurchaserDefault: true},
{Key: ModuleCollectionRules, Title: "采集规则", Path: "/collection-rules", RouteName: "GoAutoCollectionRules", Component: "/goauto/collection-rules/index", Icon: "code", Sort: 58, PurchaserDefault: true},
{Key: ModulePurchaseRules, Title: "采购规则", Path: "/purchase-rules", RouteName: "GoAutoPurchaseRules", Component: "/goauto/purchase-rules/index", Icon: "setting", Sort: 59, PurchaserHardHidden: true},
{Key: ModuleCollectionTasks, Title: "采集任务", Path: "/collection-tasks", RouteName: "GoAutoCollectionTasks", Component: "/goauto/collection-tasks/index", Icon: "list", Sort: 60, PurchaserDefault: true},
{Key: ModulePurchaseTasks, Title: "采购管理", Path: "/purchase-tasks", RouteName: "GoAutoPurchaseTasks", Component: "/goauto/purchase-tasks/index", Icon: "shopping", Sort: 61, PurchaserDefault: true},
{Key: ModuleAIMatching, Title: "AI 规格匹配", Path: "/ai-matching-settings", RouteName: "GoAutoAiMatchingSettings", Component: "/goauto/ai-matching-settings/index", Icon: "setting", Sort: 62, PurchaserHardHidden: true},
}
// GoAutoModules returns independent copies so callers cannot mutate the
@@ -144,6 +147,8 @@ func moduleKeyForAPI(path string) string {
return ModuleSYBProducts
case strings.HasPrefix(path, "/api/admin/v1/syb-shops"):
return ModuleSYBShops
case strings.HasPrefix(path, "/api/admin/v1/syb-product-filters"):
return ModuleSYBProductFilters
case strings.HasPrefix(path, "/api/admin/v1/syb-inner-codes"):
return ModuleSYBInnerCodes
case strings.HasPrefix(path, "/api/admin/v1/collection-rules"):
+5 -5
View File
@@ -10,8 +10,8 @@ import (
func TestGoAutoModulesOwnEveryAdminAPIExactlyOnce(t *testing.T) {
modules := GoAutoModules()
if len(modules) != 12 {
t.Fatalf("got %d modules, want 12", len(modules))
if len(modules) != 13 {
t.Fatalf("got %d modules, want 13", len(modules))
}
owners := make(map[string]int)
@@ -56,8 +56,8 @@ func TestGoAutoMenuSortsFitMySQLSignedTinyInt(t *testing.T) {
}
previous = module.Sort
}
if modules[0].Sort != 50 || modules[len(modules)-1].Sort != 61 {
t.Fatalf("GoAuto menu sort range = %d..%d, want 50..61", modules[0].Sort, modules[len(modules)-1].Sort)
if modules[0].Sort != 50 || modules[len(modules)-1].Sort != 62 {
t.Fatalf("GoAuto menu sort range = %d..%d, want 50..62", modules[0].Sort, modules[len(modules)-1].Sort)
}
}
@@ -72,7 +72,7 @@ func TestGoAutoMenuGroupsCoverModulesExactlyOnce(t *testing.T) {
wantOrder := [][]string{
{ModuleSYBProducts, ModuleSYBSyncRuns, ModuleSYBInnerCodes, ModuleShopeeProducts, ModulePDDProducts, ModuleCollectionTasks, ModulePurchaseTasks},
{ModuleSYBShops, ModuleCollectionRules, ModulePurchaseRules, ModuleDevices, ModuleAIMatching},
{ModuleSYBShops, ModuleSYBProductFilters, ModuleCollectionRules, ModulePurchaseRules, ModuleDevices, ModuleAIMatching},
}
seen := make(map[string]int)
for groupIndex, group := range groups {
+4
View File
@@ -77,6 +77,10 @@ var AdminAPIs = []APIPermission{
{"启停 SYB 店铺", "/api/admin/v1/syb-shops/:shopId/enabled", "PATCH", false},
{"删除 SYB 店铺", "/api/admin/v1/syb-shops/:shopId", "DELETE", false},
{"从 SYB 发现店铺", "/api/admin/v1/syb-shops/discover", "POST", false},
{"查看 SYB 商品过滤", "/api/admin/v1/syb-product-filters", "GET", true},
{"新增 SYB 商品过滤", "/api/admin/v1/syb-product-filters", "POST", false},
{"启停 SYB 商品过滤", "/api/admin/v1/syb-product-filters/:filterId/enabled", "PATCH", false},
{"删除 SYB 商品过滤", "/api/admin/v1/syb-product-filters/:filterId", "DELETE", false},
{"查看采集规则模板", "/api/admin/v1/collection-rules/templates/:templateId", "GET", true},
{"查看采集规则", "/api/admin/v1/collection-rules", "GET", true},
+8 -6
View File
@@ -15,12 +15,14 @@ func TestPurchaserPermissionMatrixHasNoDuplicates(t *testing.T) {
func TestPurchaserExcludesAdministratorOperations(t *testing.T) {
denied := map[string]bool{
"POST /api/admin/v1/devices/:deviceId/disable": true,
"POST /api/admin/v1/syb-shops/discover": true,
"POST /api/admin/v1/collection-rules": true,
"PUT /api/admin/v1/ai-matching-settings": true,
"POST /api/admin/v1/shopee-spec-auto-match/runs": true,
"GET /api/admin/v1/shopee-spec-auto-match/runs/latest": true,
"POST /api/admin/v1/devices/:deviceId/disable": true,
"POST /api/admin/v1/syb-shops/discover": true,
"POST /api/admin/v1/syb-product-filters": true,
"PATCH /api/admin/v1/syb-product-filters/:filterId/enabled": true,
"POST /api/admin/v1/collection-rules": true,
"PUT /api/admin/v1/ai-matching-settings": true,
"POST /api/admin/v1/shopee-spec-auto-match/runs": true,
"GET /api/admin/v1/shopee-spec-auto-match/runs/latest": true,
}
for _, permission := range PurchaserAPIs() {
if denied[permission.Method+" "+permission.Path] {
+1 -1
View File
@@ -33,7 +33,7 @@ func fixture(t *testing.T) (*gorm.DB, clientkey.Service) {
func TestEveryRouteIsExplicitlyScoped(t *testing.T) {
db, s := fixture(t)
routes := Inventory()
if len(s.Modules) != 12 {
if len(s.Modules) != 13 {
t.Fatal("menu groups lost")
}
router := gin.New()
+11
View File
@@ -4,6 +4,7 @@ import (
"strings"
"go-admin/app/goauto/models"
"go-admin/app/goauto/sybshop"
"gorm.io/gorm"
)
@@ -41,6 +42,7 @@ func MigratedModels() []any {
&models.SYBSpecAIParseWorkItem{},
&models.SYBSession{},
&models.SYBShop{},
&models.SYBProductFilter{},
&models.SYBSyncRun{},
&models.SYBInnerCodeRecord{},
&models.SYBInnerCodeItem{},
@@ -85,6 +87,15 @@ func Migrate(db *gorm.DB) error {
Update("task_type", models.PurchaseTaskTypeSYBOrder).Error; err != nil {
return err
}
// #269: seed immutable structure判据 and the initial keyword safety net.
seed := []struct{ kind, keyword string }{{"char", "-"}, {"char", "#"}, {"keyword", "CVC短袖220斤"}, {"keyword", "180克純棉圓領短袖"}, {"keyword", "拼色純棉短袖"}, {"keyword", "300斤牛奶絲圓領"}, {"keyword", "印花大學T"}, {"keyword", "印花帽T"}}
for _, item := range seed {
norm := sybshop.Normalize(item.keyword)
row := models.SYBProductFilter{Kind: item.kind, Keyword: item.keyword, NormalizedKeyword: norm, Enabled: true}
if err := db.Where("kind = ? AND normalized_keyword = ?", item.kind, norm).FirstOrCreate(&row).Error; err != nil {
return err
}
}
return ensureMySQLDirectSelectConstraint(db)
}
+47 -22
View File
@@ -600,32 +600,57 @@ type SYBShop struct {
func (SYBShop) TableName() string { return "syb_shop" }
// SYBProductFilter controls which SYB detail variationSku values are skipped.
// Char rows are seeded business判据 and may only be enabled/disabled; keyword
// rows are operator-maintained. DeletedFlag keeps the composite unique key
// reusable after a keyword is soft-deleted.
type SYBProductFilter struct {
ID uint64 `json:"id" gorm:"primaryKey;autoIncrement"`
Kind string `json:"kind" gorm:"size:16;not null;index;uniqueIndex:ux_syb_product_filter_key,priority:1;check:ck_syb_product_filter_kind,kind IN ('char','keyword')"`
Keyword string `json:"keyword" gorm:"type:text;not null"`
NormalizedKeyword string `json:"normalizedKeyword" gorm:"type:text;not null;uniqueIndex:ux_syb_product_filter_key,priority:2"`
Enabled bool `json:"enabled" gorm:"not null;default:true"`
LastHitAt *time.Time `json:"lastHitAt"`
LastHitCount *int `json:"lastHitCount"`
Note *string `json:"note" gorm:"type:text"`
LastChangedBy string `json:"lastChangedBy" gorm:"size:128;not null;default:''"`
LastChangedAt *time.Time `json:"lastChangedAt"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
DeletedFlag uint64 `json:"-" gorm:"not null;default:0;uniqueIndex:ux_syb_product_filter_key,priority:3"`
}
func (SYBProductFilter) TableName() string { return "syb_product_filter" }
// SYBSyncRun records one asynchronous SYB import. ActiveSlot is set only while
// a run is active; its unique index is the database-level single-flight guard
// across processes and is cleared when the run reaches a terminal state.
type SYBSyncRun struct {
ID uint64 `json:"id" gorm:"primaryKey;autoIncrement"`
DateFrom string `json:"dateFrom" gorm:"size:10;not null"`
DateTo string `json:"dateTo" gorm:"size:10;not null"`
Status string `json:"status" gorm:"size:16;not null;index"`
ActiveSlot *uint8 `json:"-" gorm:"uniqueIndex:ux_syb_sync_run_active_slot"`
DaysTotal int `json:"daysTotal" gorm:"not null;default:0"`
DaysProcessed int `json:"daysProcessed" gorm:"not null;default:0"`
OrderCount int `json:"orderCount" gorm:"not null;default:0"`
DetailCount int `json:"detailCount" gorm:"not null;default:0"`
AcceptedCount int `json:"acceptedCount" gorm:"not null;default:0"`
ShopSkipped int `json:"shopSkipped" gorm:"not null;default:0"`
Created int `json:"created" gorm:"column:created_count;not null;default:0"`
Updated int `json:"updated" gorm:"column:updated_count;not null;default:0"`
ShopFilterHash string `json:"shopFilterHash" gorm:"size:64;not null"`
ShopBreakdownJSON string `json:"-" gorm:"type:text;not null"`
ErrorMessage string `json:"errorMessage" gorm:"size:1000;not null;default:''"`
OperatorID uint64 `json:"operatorId" gorm:"not null;default:0"`
OperatorName string `json:"operatorName" gorm:"size:128;not null;default:''"`
StartedAt time.Time `json:"startedAt" gorm:"not null"`
FinishedAt *time.Time `json:"finishedAt"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID uint64 `json:"id" gorm:"primaryKey;autoIncrement"`
DateFrom string `json:"dateFrom" gorm:"size:10;not null"`
DateTo string `json:"dateTo" gorm:"size:10;not null"`
Status string `json:"status" gorm:"size:16;not null;index"`
ActiveSlot *uint8 `json:"-" gorm:"uniqueIndex:ux_syb_sync_run_active_slot"`
DaysTotal int `json:"daysTotal" gorm:"not null;default:0"`
DaysProcessed int `json:"daysProcessed" gorm:"not null;default:0"`
OrderCount int `json:"orderCount" gorm:"not null;default:0"`
DetailCount int `json:"detailCount" gorm:"not null;default:0"`
AcceptedCount int `json:"acceptedCount" gorm:"not null;default:0"`
ShopSkipped int `json:"shopSkipped" gorm:"not null;default:0"`
CharFilterSkipped int `json:"charFilterSkipped" gorm:"column:char_filter_skipped;not null;default:0"`
KeywordFilterSkipped int `json:"keywordFilterSkipped" gorm:"column:keyword_filter_skipped;not null;default:0"`
Created int `json:"created" gorm:"column:created_count;not null;default:0"`
Updated int `json:"updated" gorm:"column:updated_count;not null;default:0"`
ShopFilterHash string `json:"shopFilterHash" gorm:"size:64;not null"`
ShopBreakdownJSON string `json:"-" gorm:"type:text;not null"`
ErrorMessage string `json:"errorMessage" gorm:"size:1000;not null;default:''"`
OperatorID uint64 `json:"operatorId" gorm:"not null;default:0"`
OperatorName string `json:"operatorName" gorm:"size:128;not null;default:''"`
StartedAt time.Time `json:"startedAt" gorm:"not null"`
FinishedAt *time.Time `json:"finishedAt"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
func (SYBSyncRun) TableName() string { return "syb_sync_run" }
+27 -4
View File
@@ -11,6 +11,7 @@ import (
"time"
"go-admin/app/goauto/sybclient"
"go-admin/app/goauto/sybproductfilter"
"go-admin/app/goauto/sybshop"
"gorm.io/gorm"
@@ -52,8 +53,10 @@ type SyncReport struct {
// ShopSkipped how many did not. OrderCount stays the raw total: keeping the
// unfiltered number visible is what makes a shop missing from the
// allow-list obvious rather than silent.
AcceptedCount int `json:"acceptedCount"`
ShopSkipped int `json:"shopSkipped"`
AcceptedCount int `json:"acceptedCount"`
ShopSkipped int `json:"shopSkipped"`
CharFilterSkipped int `json:"charFilterSkipped"`
KeywordFilterSkipped int `json:"keywordFilterSkipped"`
// ShopBreakdown separates accepted and skipped orders for every shop. Without
// it a renamed or newly opened shop is never noticed (#50).
ShopBreakdown map[string]ShopBreakdown `json:"shopBreakdown"`
@@ -128,6 +131,10 @@ func SyncWithShopSnapshot(ctx context.Context, db *gorm.DB, client *sybclient.Cl
if len(allowed) == 0 {
return report, ErrNoEnabledShop
}
filters, err := sybproductfilter.LoadEnabled(ctx, db)
if err != nil {
return report, fmt.Errorf("读取商品过滤规则失败: %w", err)
}
report.ShopFilterHash = enabledShopHash(allowed)
emit := func(daysProcessed int) error {
if progress == nil {
@@ -181,7 +188,7 @@ func SyncWithShopSnapshot(ctx context.Context, db *gorm.DB, client *sybclient.Cl
}
_, dayErr := loadDailyList(ctx, client, plan.date, pageSize, plan.total, func(rows []sybclient.StockRow, page int) error {
report.OrderCount += len(rows)
err := importSyncPage(ctx, db, client, rows, allowed, &report)
err := importSyncPage(ctx, db, client, rows, allowed, &report, filters)
if err != nil {
return fmt.Errorf("%s 第 %d 页明细/入库阶段失败: %w", plan.date, page, err)
}
@@ -204,6 +211,9 @@ func SyncWithShopSnapshot(ctx context.Context, db *gorm.DB, client *sybclient.Cl
}
}
report.FinishedAt = time.Now().UTC()
if err := sybproductfilter.UpdateHits(ctx, db, filters, report.CharFilterSkipped, report.KeywordFilterSkipped, report.FinishedAt); err != nil {
return report, err
}
return report, errors.Join(failures...)
}
@@ -347,7 +357,11 @@ func validateDetailBatch(requested []int64, details []sybclient.StockDetail) err
// applyStockDetail writes one shipment order's items. Each item is its own
// ApplyDetail call and its own transaction — a several-thousand-row sync must
// not sit in one long-held transaction.
func applyStockDetail(ctx context.Context, db *gorm.DB, row sybclient.StockRow, detail sybclient.StockDetail, report *SyncReport) error {
func applyStockDetail(ctx context.Context, db *gorm.DB, row sybclient.StockRow, detail sybclient.StockDetail, report *SyncReport, sets ...sybproductfilter.RuleSet) error {
var filters sybproductfilter.RuleSet
if len(sets) > 0 {
filters = sets[0]
}
order := OrderInput{
Code: detail.Code,
StockID: uint64(detail.ID),
@@ -357,6 +371,15 @@ func applyStockDetail(ctx context.Context, db *gorm.DB, row sybclient.StockRow,
order.Code = row.Code
}
for _, item := range detail.Details {
variation := stringField(item.Raw, "variationSku")
if charHit, keywordHit := filters.Match(variation); charHit || keywordHit {
if charHit {
report.CharFilterSkipped++
} else {
report.KeywordFilterSkipped++
}
continue
}
raw, err := json.Marshal(item.Raw)
if err != nil {
return fmt.Errorf("货运单 %s 明细 %d 原始数据编码失败: %w", order.Code, item.ID, err)
+7 -2
View File
@@ -3,6 +3,7 @@ package sybimport
import (
"context"
"go-admin/app/goauto/sybclient"
"go-admin/app/goauto/sybproductfilter"
"go-admin/app/goauto/sybshop"
"gorm.io/gorm"
)
@@ -14,7 +15,11 @@ func (e *syncFatalError) Unwrap() error { return e.error }
// Read all remote details before opening a page transaction. Publish counters
// only after commit, so a rollback never reports records as saved.
func importSyncPage(ctx context.Context, db *gorm.DB, client *sybclient.Client, rows []sybclient.StockRow, allowed map[string]string, report *SyncReport) error {
func importSyncPage(ctx context.Context, db *gorm.DB, client *sybclient.Client, rows []sybclient.StockRow, allowed map[string]string, report *SyncReport, sets ...sybproductfilter.RuleSet) error {
var filters sybproductfilter.RuleSet
if len(sets) > 0 {
filters = sets[0]
}
next := *report
next.ShopBreakdown = make(map[string]ShopBreakdown, len(report.ShopBreakdown))
for k, v := range report.ShopBreakdown {
@@ -69,7 +74,7 @@ func importSyncPage(ctx context.Context, db *gorm.DB, client *sybclient.Client,
next.ShopSkipped++
continue
}
if err := applyStockDetail(ctx, tx, byID[detail.ID], detail, &next); err != nil {
if err := applyStockDetail(ctx, tx, byID[detail.ID], detail, &next, filters); err != nil {
return err
}
}
+2
View File
@@ -70,6 +70,7 @@ func (s *SyncRunService) UpdateProgress(ctx context.Context, id uint64, progress
"days_total": progress.DaysTotal, "days_processed": progress.DaysProcessed,
"order_count": progress.Report.OrderCount, "detail_count": progress.Report.DetailCount,
"accepted_count": progress.Report.AcceptedCount, "shop_skipped": progress.Report.ShopSkipped,
"char_filter_skipped": progress.Report.CharFilterSkipped, "keyword_filter_skipped": progress.Report.KeywordFilterSkipped,
"created_count": progress.Report.Created, "updated_count": progress.Report.Updated,
"shop_filter_hash": progress.Report.ShopFilterHash, "shop_breakdown_json": payload,
}).Error
@@ -92,6 +93,7 @@ func (s *SyncRunService) Finish(ctx context.Context, id uint64, status string, r
"status": status, "active_slot": nil, "finished_at": now, "error_message": message,
"order_count": report.OrderCount, "detail_count": report.DetailCount,
"accepted_count": report.AcceptedCount, "shop_skipped": report.ShopSkipped,
"char_filter_skipped": report.CharFilterSkipped, "keyword_filter_skipped": report.KeywordFilterSkipped,
"created_count": report.Created, "updated_count": report.Updated,
"shop_breakdown_json": payload,
}
@@ -0,0 +1,129 @@
package sybproductfilter
import (
"encoding/json"
"errors"
"github.com/gin-gonic/gin"
"github.com/go-admin-team/go-admin-core/sdk/pkg"
jwt "github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth"
"gorm.io/gorm"
"net/http"
"strconv"
)
type Handler struct{ DB *gorm.DB }
func (h Handler) service(c *gin.Context) (*Service, bool) {
db := h.DB
var e error
if db == nil {
db, e = pkg.GetOrm(c)
}
if e != nil {
writeError(c, internal(e))
return nil, false
}
return NewService(db), true
}
func (h Handler) List(c *gin.Context) {
s, ok := h.service(c)
if !ok {
return
}
p, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
ps, _ := strconv.Atoi(c.DefaultQuery("pageSize", "50"))
r, e := s.List(c.Request.Context(), ListRequest{Page: p, PageSize: ps, Kind: c.Query("kind"), Keyword: c.Query("keyword"), EnabledOnly: c.Query("enabledOnly") == "true"})
if e != nil {
writeError(c, e)
return
}
c.JSON(200, gin.H{"code": 200, "data": r})
}
func (h Handler) Create(c *gin.Context) {
var r CreateRequest
if !decode(c, &r) {
return
}
s, ok := h.service(c)
if !ok {
return
}
x, e := s.Create(c.Request.Context(), r)
if e != nil {
writeError(c, e)
return
}
c.JSON(200, gin.H{"code": 200, "data": x})
}
func (h Handler) SetEnabled(c *gin.Context) {
id, ok := idParam(c)
if !ok {
return
}
var r SetEnabledRequest
if !decode(c, &r) {
return
}
s, ok := h.service(c)
if !ok {
return
}
claims := jwt.ExtractClaims(c)
name, _ := claims["nice"].(string)
x, e := s.SetEnabled(c.Request.Context(), id, r, name)
if e != nil {
writeError(c, e)
return
}
c.JSON(200, gin.H{"code": 200, "data": x})
}
func (h Handler) Delete(c *gin.Context) {
id, ok := idParam(c)
if !ok {
return
}
s, ok := h.service(c)
if !ok {
return
}
if e := s.Delete(c.Request.Context(), id); e != nil {
writeError(c, e)
return
}
c.JSON(200, gin.H{"code": 200, "data": gin.H{"deleted": true}})
}
func idParam(c *gin.Context) (uint64, bool) {
id, e := strconv.ParseUint(c.Param("filterId"), 10, 64)
if e != nil || id == 0 {
writeError(c, invalid("过滤规则 id 必须是正整数"))
return 0, false
}
return id, true
}
func decode(c *gin.Context, t any) bool {
d := json.NewDecoder(c.Request.Body)
d.DisallowUnknownFields()
if e := d.Decode(t); e != nil {
writeError(c, invalid("请求体解析失败:"+e.Error()))
return false
}
return true
}
func writeError(c *gin.Context, e error) {
var se *ServiceError
if !errors.As(e, &se) {
se = internal(e).(*ServiceError)
}
status := 500
switch se.Code {
case CodeInvalidRequest:
status = 422
case CodeConflict:
status = 409
case CodeNotFound:
status = 404
case CodeForbidden:
status = http.StatusForbidden
}
c.JSON(status, gin.H{"code": se.Code, "message": se.Message})
}
@@ -0,0 +1,15 @@
package sybproductfilter
import (
"github.com/gin-gonic/gin"
jwt "github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth"
"go-admin/common/middleware"
)
func InitRouter(e *gin.Engine, a *jwt.GinJWTMiddleware) {
g := e.Group("/api/admin/v1/syb-product-filters").Use(a.MiddlewareFunc()).Use(middleware.AuthCheckRole())
g.GET("", Handler{}.List)
g.POST("", middleware.RequireRoleKey("admin"), Handler{}.Create)
g.PATCH("/:filterId/enabled", middleware.RequireRoleKey("admin"), Handler{}.SetEnabled)
g.DELETE("/:filterId", middleware.RequireRoleKey("admin"), Handler{}.Delete)
}
@@ -0,0 +1,223 @@
package sybproductfilter
import (
"context"
"errors"
"fmt"
"strings"
"time"
"go-admin/app/goauto/models"
"go-admin/app/goauto/sybshop"
"gorm.io/gorm"
)
type ServiceError struct {
Code, Message string
Cause error
}
func (e *ServiceError) Error() string {
if e.Cause != nil {
return fmt.Sprintf("%s: %v", e.Message, e.Cause)
}
return e.Message
}
func (e *ServiceError) Unwrap() error { return e.Cause }
const (
CodeInvalidRequest = "INVALID_REQUEST"
CodeNotFound = "SYB_PRODUCT_FILTER_NOT_FOUND"
CodeConflict = "SYB_PRODUCT_FILTER_DUPLICATE"
CodeForbidden = "FORBIDDEN"
)
func invalid(m string) error { return &ServiceError{Code: CodeInvalidRequest, Message: m} }
func internal(e error) error {
return &ServiceError{Code: "INTERNAL_ERROR", Message: "服务器内部错误", Cause: e}
}
type Service struct{ DB *gorm.DB }
func NewService(db *gorm.DB) *Service { return &Service{DB: db} }
type ListRequest struct {
Page, PageSize int
Kind, Keyword string
EnabledOnly bool
}
type ListResponse struct {
Items []models.SYBProductFilter `json:"items"`
Total int64 `json:"total"`
EnabledCount int64 `json:"enabledCount"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
}
func (s *Service) List(ctx context.Context, r ListRequest) (ListResponse, error) {
if r.Page < 1 {
r.Page = 1
}
if r.PageSize < 1 || r.PageSize > 200 {
r.PageSize = 50
}
q := s.DB.WithContext(ctx).Model(&models.SYBProductFilter{})
if r.Kind != "" {
q = q.Where("kind = ?", r.Kind)
}
if k := sybshop.Normalize(r.Keyword); k != "" {
q = q.Where("normalized_keyword LIKE ?", "%"+k+"%")
}
if r.EnabledOnly {
q = q.Where("enabled = ?", true)
}
var total, enabled int64
if err := q.Count(&total).Error; err != nil {
return ListResponse{}, internal(err)
}
items := make([]models.SYBProductFilter, 0, r.PageSize)
if err := q.Order("kind ASC, id ASC").Offset((r.Page - 1) * r.PageSize).Limit(r.PageSize).Find(&items).Error; err != nil {
return ListResponse{}, internal(err)
}
if err := s.DB.WithContext(ctx).Model(&models.SYBProductFilter{}).Where("enabled = ?", true).Count(&enabled).Error; err != nil {
return ListResponse{}, internal(err)
}
return ListResponse{Items: items, Total: total, EnabledCount: enabled, Page: r.Page, PageSize: r.PageSize}, nil
}
type CreateRequest struct {
Kind string `json:"kind"`
Keyword string `json:"keyword"`
Enabled *bool `json:"enabled"`
Note *string `json:"note"`
}
func validate(kind, keyword string) (string, string, error) {
if kind != "char" && kind != "keyword" {
return "", "", invalid("kind 必须是 char 或 keyword")
}
keyword = strings.TrimSpace(keyword)
if keyword == "" {
return "", "", invalid("关键词不能为空")
}
if sybshop.ContainsControl(keyword) {
return "", "", invalid("关键词不能包含控制字符")
}
return keyword, sybshop.Normalize(keyword), nil
}
func (s *Service) Create(ctx context.Context, r CreateRequest) (models.SYBProductFilter, error) {
if r.Kind == "char" {
return models.SYBProductFilter{}, &ServiceError{Code: CodeForbidden, Message: "结构过滤不可新增"}
}
k, n, e := validate(r.Kind, r.Keyword)
if e != nil {
return models.SYBProductFilter{}, e
}
var x models.SYBProductFilter
if err := s.DB.WithContext(ctx).Where("kind = ? AND normalized_keyword = ?", r.Kind, n).First(&x).Error; err == nil {
return models.SYBProductFilter{}, &ServiceError{Code: CodeConflict, Message: fmt.Sprintf("该关键词已存在(归一化后与「%s」相同)", x.Keyword)}
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
return models.SYBProductFilter{}, internal(err)
}
en := true
if r.Enabled != nil {
en = *r.Enabled
}
x = models.SYBProductFilter{Kind: r.Kind, Keyword: k, NormalizedKeyword: n, Enabled: en, Note: r.Note}
if err := s.DB.WithContext(ctx).Create(&x).Error; err != nil {
return models.SYBProductFilter{}, internal(err)
}
return x, nil
}
type SetEnabledRequest struct {
Enabled bool `json:"enabled"`
}
func (s *Service) SetEnabled(ctx context.Context, id uint64, r SetEnabledRequest, operator string) (models.SYBProductFilter, error) {
var x models.SYBProductFilter
if err := s.DB.WithContext(ctx).First(&x, id).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return x, &ServiceError{Code: CodeNotFound, Message: "过滤规则不存在"}
}
return x, internal(err)
}
now := time.Now().UTC()
updates := map[string]any{"enabled": r.Enabled, "updated_at": now}
if x.Kind == "char" {
updates["last_changed_by"] = strings.TrimSpace(operator)
updates["last_changed_at"] = now
}
if err := s.DB.WithContext(ctx).Model(&x).Updates(updates).Error; err != nil {
return x, internal(err)
}
x.Enabled = r.Enabled
x.UpdatedAt = now
if x.Kind == "char" {
x.LastChangedBy = strings.TrimSpace(operator)
x.LastChangedAt = &now
}
return x, nil
}
func (s *Service) Delete(ctx context.Context, id uint64) error {
return s.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var x models.SYBProductFilter
if err := tx.First(&x, id).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return &ServiceError{Code: CodeNotFound, Message: "过滤规则不存在"}
}
return internal(err)
}
if x.Kind == "char" {
return &ServiceError{Code: CodeForbidden, Message: "结构过滤不可删除"}
}
return tx.Model(&x).Updates(map[string]any{"deleted_flag": x.ID, "deleted_at": time.Now().UTC()}).Error
})
}
type RuleSet struct{ Chars, Keywords []models.SYBProductFilter }
func LoadEnabled(ctx context.Context, db *gorm.DB) (RuleSet, error) {
var rows []models.SYBProductFilter
if err := db.WithContext(ctx).Where("enabled = ?", true).Find(&rows).Error; err != nil {
return RuleSet{}, err
}
r := RuleSet{}
for _, x := range rows {
if x.Kind == "char" {
r.Chars = append(r.Chars, x)
} else {
r.Keywords = append(r.Keywords, x)
}
}
return r, nil
}
func (r RuleSet) Match(v string) (char, keyword bool) {
if v == "" {
return false, false
}
n := sybshop.Normalize(v)
for _, x := range r.Chars {
if strings.Contains(n, x.NormalizedKeyword) {
return true, false
}
}
for _, x := range r.Keywords {
if strings.Contains(n, x.NormalizedKeyword) {
return false, true
}
}
return false, false
}
func UpdateHits(ctx context.Context, db *gorm.DB, r RuleSet, charHits, keywordHits int, at time.Time) error {
for _, x := range append(r.Chars, r.Keywords...) {
count := charHits
if x.Kind == "keyword" {
count = keywordHits
}
if err := db.WithContext(ctx).Model(&models.SYBProductFilter{}).Where("id = ?", x.ID).Updates(map[string]any{"last_hit_count": count, "last_hit_at": at}).Error; err != nil {
return err
}
}
return nil
}
@@ -0,0 +1,61 @@
package sybproductfilter
import (
"context"
"go-admin/app/goauto/migrations"
"go-admin/app/goauto/models"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"testing"
)
func testDB(t *testing.T) *gorm.DB {
db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err = migrations.Migrate(db); err != nil {
t.Fatal(err)
}
return db
}
func TestMatchORAndNormalization(t *testing.T) {
r := RuleSet{Chars: []models.SYBProductFilter{{NormalizedKeyword: "-"}, {NormalizedKeyword: "#"}}, Keywords: []models.SYBProductFilter{{NormalizedKeyword: "cvc短袖220斤"}}}
for _, v := range []string{"DD#004", "300斤牛奶絲圓領#A057"} {
c, k := r.Match(v)
if !c || k {
t.Fatalf("%q should char-match", v)
}
}
if c, k := r.Match(""); c || k {
t.Fatal("empty variationSku must not match")
}
if c, k := r.Match(" CVC短袖220斤 "); c || !k {
t.Fatal("full-width/space keyword should match")
}
if _, k := r.Match("CVC短袖220斤"); k != true {
t.Fatal("lowercase keyword should match")
}
if _, k := r.Match("純棉"); k {
t.Fatal("normalization must not simplify traditional Chinese")
}
}
func TestCharCreateDeleteRejectedAndKeywordDuplicate(t *testing.T) {
db := testDB(t)
s := NewService(db)
ctx := context.Background()
if _, e := s.Create(ctx, CreateRequest{Kind: "char", Keyword: "@"}); e == nil {
t.Fatal("char create must be rejected")
}
if e := s.Delete(ctx, 1); e == nil {
t.Fatal("seed char delete must be rejected")
}
if _, e := s.Create(ctx, CreateRequest{Kind: "keyword", Keyword: " CVC "}); e != nil {
t.Fatal(e)
}
if _, e := s.Create(ctx, CreateRequest{Kind: "keyword", Keyword: "cvc"}); e == nil {
t.Fatal("normalized duplicate must be rejected")
}
}
@@ -0,0 +1,6 @@
import request from '@/utils/request'
const base = '/api/admin/v1/syb-product-filters'
export function listSybProductFilters(params) { return request({ url: base, method: 'get', params }) }
export function createSybProductFilter(data) { return request({ url: base, method: 'post', data }) }
export function setSybProductFilterEnabled(id, data) { return request({ url: `${base}/${id}/enabled`, method: 'patch', data }) }
export function deleteSybProductFilter(id) { return request({ url: `${base}/${id}`, method: 'delete' }) }
@@ -0,0 +1,19 @@
<template>
<BasicLayout><template #wrapper><el-card class="page-card" shadow="never">
<el-alert v-if="!isAdmin" title="当前账号只有查看权限。新增、编辑、启用、停用和删除关键词需要管理员操作。" type="info" :closable="false" show-icon class="notice" />
<el-alert title="关键词匹配 variationSku,不匹配商品标题。" type="info" :closable="false" show-icon class="notice" />
<section><h3>结构过滤(不可增删,仅可停用)</h3><p class="muted">这两个字符是识别档口商品的判据,命中即不入库。需要变更判据请另行建单评估。</p>
<el-table v-loading="loading" :data="chars" border stripe><el-table-column label="字符" prop="keyword" width="180"/><el-table-column label="状态" width="150"><template #default="{row}"><el-switch v-if="isAdmin" v-model="row.enabled" @change="toggle(row)"/><el-tag v-else :type="row.enabled?'success':'info'">{{row.enabled?'已启用':'已停用'}}</el-tag></template></el-table-column><el-table-column label="最近命中" width="140"><template #default="{row}">{{ row.lastHitCount == null ? '尚未同步' : row.lastHitCount }}</template></el-table-column><el-table-column label="最近变更" min-width="220"><template #default="{row}">{{ row.lastChangedAt ? `${format(row.lastChangedAt)} ${row.lastChangedBy || ''} ${row.enabled ? '启用' : '停用'}` : '尚未记录' }}</template></el-table-column><el-table-column label="操作" width="110"><template #default><span class="muted">不可删除</span></template></el-table-column></el-table></section>
<section class="keyword-section"><div class="section-head"><div><h3>关键词过滤</h3><p class="muted">在结构过滤之后执行,命中数为结构过滤未覆盖的部分。</p></div><el-button v-if="isAdmin" type="primary" @click="editor.open=true">新增关键词</el-button></div>
<el-table v-loading="loading" :data="keywords" border stripe empty-text="还没有配置过滤关键词。结构过滤(variationSku 含 - 或 #)仍然生效,本页只影响额外的关键词过滤。"><el-table-column label="关键词" prop="keyword" min-width="260"/><el-table-column label="状态" width="140"><template #default="{row}"><el-switch v-if="isAdmin" v-model="row.enabled" @change="toggle(row)"/><el-tag v-else :type="row.enabled?'success':'info'">{{row.enabled?'已启用':'已停用'}}</el-tag></template></el-table-column><el-table-column label="最近命中" width="130"><template #default="{row}">{{ row.lastHitCount == null ? '尚未同步' : row.lastHitCount }}</template></el-table-column><el-table-column label="备注" prop="note" min-width="180"><template #default="{row}">{{ row.note || '—' }}</template></el-table-column><el-table-column label="操作" width="100"><template #default="{row}"><el-button v-if="isAdmin" type="danger" link @click="remove(row)">删除</el-button><span v-else class="muted">只读</span></template></el-table-column></el-table></section>
<p class="scope-note">停用或删除只影响后续同步,已经入库的商品明细不会被删除。</p>
<el-dialog v-model="editor.open" title="新增关键词" width="520px"><el-form label-position="top"><el-form-item label="关键词"><el-input v-model="editor.keyword" maxlength="200" /></el-form-item><el-form-item label="备注"><el-input v-model="editor.note" /></el-form-item></el-form><template #footer><el-button @click="editor.open=false">取消</el-button><el-button type="primary" @click="save">保存</el-button></template></el-dialog>
</el-card></template></BasicLayout>
</template>
<script>
import { ElMessage, ElMessageBox } from 'element-plus'
import { listSybProductFilters, createSybProductFilter, setSybProductFilterEnabled, deleteSybProductFilter } from '@/api/goauto/syb-product-filters'
export default { name:'GoAutoSybProductFilters', data(){return {loading:false,chars:[],keywords:[],editor:{open:false,keyword:'',note:''}}}, computed:{isAdmin(){return (this.$store.getters.roles||[]).includes('admin')}}, created(){this.load()}, methods:{ async load(){this.loading=true;try{const r=await listSybProductFilters({page:1,pageSize:200});const rows=r.data.items||[];this.chars=rows.filter(x=>x.kind==='char');this.keywords=rows.filter(x=>x.kind==='keyword')}finally{this.loading=false}}, format(v){return new Intl.DateTimeFormat('zh-CN',{dateStyle:'medium',timeStyle:'short'}).format(new Date(v))}, async toggle(row){if(row.kind==='char'&&!row.enabled){await ElMessageBox.confirm(`停用「${row.keyword}」后,SYB 同步将不再过滤含 ${row.keyword} 的档口商品。按最近一次同步数据估算,约 ${row.lastHitCount||0} 条明细会恢复入库。确认停用?`,'确认停用结构过滤',{type:'warning'}).catch(()=>{row.enabled=true;throw new Error('cancel')})}try{await setSybProductFilterEnabled(row.id,{enabled:row.enabled});ElMessage.success(row.enabled?'规则已启用':'规则已停用')}catch(e){row.enabled=!row.enabled; if(e.message!=='cancel')throw e}},async save(){if(!this.editor.keyword.trim())return;await createSybProductFilter({kind:'keyword',keyword:this.editor.keyword,note:this.editor.note||undefined});ElMessage.success('关键词已添加');this.editor={open:false,keyword:'',note:''};this.load()},async remove(row){await ElMessageBox.confirm(`删除关键词「${row.keyword}」?`,'删除关键词',{type:'warning'});await deleteSybProductFilter(row.id);ElMessage.success('关键词已删除');this.load()}}
}
</script>
<style lang="scss" scoped>.page-card{min-height:calc(100vh - 124px)}.notice{margin-bottom:16px}section{margin-bottom:28px}h3{margin:0 0 8px}.muted,.scope-note{font-size:12px;color:#606266}.section-head{display:flex;justify-content:space-between;align-items:center}.keyword-section{border-top:1px solid #ebeef5;padding-top:20px}.scope-note{margin-top:12px}</style>
@@ -48,6 +48,8 @@
<el-descriptions-item label="商品明细">{{ detail.item.detailCount }} 条</el-descriptions-item>
<el-descriptions-item label="符合店铺">{{ detail.item.acceptedCount }} 张</el-descriptions-item>
<el-descriptions-item label="跳过店铺">{{ detail.item.shopSkipped }} 张</el-descriptions-item>
<el-descriptions-item label="结构过滤命中">{{ detail.item.charFilterSkipped || 0 }} 条</el-descriptions-item>
<el-descriptions-item label="关键词过滤命中">{{ detail.item.keywordFilterSkipped || 0 }} 条</el-descriptions-item>
<el-descriptions-item label="新增 / 覆盖">{{ detail.item.created }} / {{ detail.item.updated }}</el-descriptions-item>
<el-descriptions-item label="操作人">{{ detail.item.operatorName || `用户 ${detail.item.operatorId || '-'}` }}</el-descriptions-item>
<el-descriptions-item label="开始时间">{{ formatTime(detail.item.startedAt) }}</el-descriptions-item>