fix(server): SYB 结构过滤合并为一条,要求同时包含 - 和 # (#286)

结构过滤原为两条 char 规则(- 和 #),RuleSet.Match 逐条 Contains、
命中即返回,是或关系。线上最近一次完整同步(syb_sync_run 541)里
- 命中 384、# 命中 148,合计 532 条被过滤;因 - 先判且命中即返回,
那 148 条是含 # 但不含 - 的。

改为一条,keyword 存必需字符集合 -#,匹配要求集合中每个字符都出现:

- 同时含 - 和 # → 过滤
- 只含其一 → 放行
- 都不含 → 放行

已向用户说明每次同步至少多放行 148 条,用户确认后实施。

`[必须]` migrations/migrate.go 的种子必须同步改成一条。该函数每次启动
都会 FirstOrCreate,若继续播种旧的两条,下次重启就会把迁移
1789500000000 合并掉的行重新建回来,过滤静默退回或关系。已加
TestFreshDatabaseSeedsOneStructuralRule 锁住。

关键词过滤不变,仍是子串匹配,并加测试防止字符集合语义误用到关键词上。

连带效果:真实样本 300斤牛奶絲圓領#A057 只含 #,不再被结构过滤命中后
落到关键词规则上仍被过滤——#269 把关键词称为 forward safety net,这是
它第一次真的接住东西。realworld 用例的统计由 char=5/keyword=0/kept=6
变为 char=3/keyword=1/kept=7。

app/goauto/sybimport 的 12 个失败先于本次存在(#285),未新增。

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NTDbDcwbDw1TSAcE6wfh2F
This commit is contained in:
QiuSW
2026-09-15 17:15:40 +08:00
co-authored by Claude Opus 5
parent 95b5f05ae8
commit 174d92a1c9
6 changed files with 180 additions and 35 deletions
+6 -1
View File
@@ -88,7 +88,12 @@ func Migrate(db *gorm.DB) error {
return err
}
// #269: seed immutable structure判据 and the initial keyword safety net.
seed := []struct{ kind, keyword string }{{"char", "-"}, {"char", "#"}, {"keyword", "CVC短袖220斤"}, {"keyword", "180克純棉圓領短袖"}, {"keyword", "拼色純棉短袖"}, {"keyword", "300斤牛奶絲圓領"}, {"keyword", "印花大學T"}, {"keyword", "印花帽T"}}
//
// `[必须]` 结构过滤自 #286 起只有**一条**,keyword 存的是必需字符集合
// `-#`,意思是“同时包含 - 和 #”。这里不能再播种旧的两条:本函数
// 每次启动都会 FirstOrCreate,播了就会把迁移 1789500000000 合并掉的
// `-` 和 `#` 重新建回来,过滤静默退回或关系。
seed := []struct{ kind, keyword string }{{"char", "-#"}, {"keyword", "CVC短袖220斤"}, {"keyword", "180克純棉圓領短袖"}, {"keyword", "拼色純棉短袖"}, {"keyword", "300斤牛奶絲圓領"}, {"keyword", "印花大學T"}, {"keyword", "印花帽T"}}
for _, item := range seed {
norm := sybshop.Normalize(item.keyword)
row := models.SYBProductFilter{Kind: item.kind, Keyword: item.keyword, NormalizedKeyword: norm, Enabled: true}
@@ -20,9 +20,12 @@ func TestRealWorldSamplesPerRuleCounting(t *testing.T) {
}{
{"新寮-雅伊阁大码女装#301", "char"},
{"广达-爆款大码女装#9018", "char"},
{"DD#004", "char"},
{"300斤牛奶絲圓領#A057", "char"},
{"上寮-琪琪尚雅网批#2604##(&15)", "char"},
// 只含 # 不含 -,#286 起不再被结构过滤。
{"DD#004", ""},
// 同样不再被结构过滤,但落到了关键词规则上——#269 把关键词称为
// "forward safety net",这里正是它第一次真的接住东西。
{"300斤牛奶絲圓領#A057", "keyword"},
{"PDD0582", ""},
{"my238", ""},
{"le171", ""},
@@ -51,7 +54,9 @@ func TestRealWorldSamplesPerRuleCounting(t *testing.T) {
kwN++
}
}
if charN != 5 || kwN != 0 || keptN != 6 {
// #286:结构过滤由或改与,5 条 char 命中变为 3 条;其中一条转由关键词接住,
// 另一条重新入库。
if charN != 3 || kwN != 1 || keptN != 7 {
t.Fatalf("统计不符:char=%d keyword=%d kept=%d", charN, kwN, keptN)
}
if err := UpdateHits(ctx, db, set, hits, time.Now().UTC()); err != nil {
@@ -68,17 +73,32 @@ func TestRealWorldSamplesPerRuleCounting(t *testing.T) {
}
got[x.Keyword] = *x.LastHitCount
}
// "#" 命中 4 条(301/9018/DD#004/A057 ... 其中带 - 的先被 "-" 规则接走)
if got["-"]+got["#"] != 5 {
t.Fatalf("两条结构规则命中数之和应为 5,实际 - =%d # =%d", got["-"], got["#"])
// #286:结构过滤只剩一条必需字符集合 `-#`,应记下 3 条命中。
if got["-#"] != 3 {
t.Fatalf("结构规则命中数应为 3,实际 %d", got["-#"])
}
if got["-"] == got["#"] {
t.Fatalf("两条规则不应记成同一个数(这正是 #269 修的 bug): - =%d # =%d", got["-"], got["#"])
if len(got) != 1 {
t.Fatalf("结构过滤应当只有一条,实际 %v", got)
}
// 关键词仍然各记各的(#269 修的那个 bug 不得回退):
// 300斤牛奶絲圓領 这条接住了 1 条,其余关键词应为 0。
kwHits := map[string]int{}
for _, x := range after.Keywords {
if x.LastHitCount == nil || *x.LastHitCount != 0 {
t.Fatalf("未命中的关键词 %q 应记 0,实际 %v", x.Keyword, x.LastHitCount)
if x.LastHitCount == nil {
t.Fatalf("关键词 %q 必须被写入命中数", x.Keyword)
}
kwHits[x.Keyword] = *x.LastHitCount
}
if kwHits["300斤牛奶絲圓領"] != 1 {
t.Fatalf("关键词 300斤牛奶絲圓領 应接住 1 条,实际 %d", kwHits["300斤牛奶絲圓領"])
}
for k, n := range kwHits {
if k == "300斤牛奶絲圓領" {
continue
}
if n != 0 {
t.Fatalf("未命中的关键词 %q 应记 0,实际 %d", k, n)
}
}
t.Logf("结构规则命中:- =%d # =%d", got["-"], got["#"])
t.Logf("结构规则 -# 命中 %d;关键词命中 %v", got["-#"], kwHits)
}
+28 -6
View File
@@ -198,22 +198,44 @@ func LoadEnabled(ctx context.Context, db *gorm.DB) (RuleSet, error) {
// It returns the matching rule rather than a pair of booleans because the
// caller has to attribute the hit to one specific rule: every rule keeps its
// own LastHitCount, and the disable-confirmation dialog quotes that per-rule
// number ("停用「#」会让约 N 条恢复入库"). Collapsing the result to
// char/keyword booleans would force every rule of a kind to share one
// aggregate, and the dialog would then warn with the wrong figure — on real
// data "#" matches 8498 rows and "-" matches 6392, so the two must never be
// reported as the same number.
// number. Collapsing the result to char/keyword booleans would force every
// rule of a kind to share one aggregate, and the dialog would then warn with
// the wrong figure.
//
// 自 #286 起结构过滤只有一条(必需字符集合 `-#`),但按规则记数的机制保持不变:
// 关键词规则仍然各自独立计数。
//
// Structural rules are checked before keyword rules so a hit is attributed the
// way the page presents the two sections, and an empty variationSku never
// matches: it is a product with no supplier code yet, not a 档口 item.
// containsAllRunes reports whether every rune of set appears somewhere in v.
//
// `[必须]` 这是结构过滤与关键词过滤的关键差别。关键词是子串匹配;结构过滤的
// keyword 存的是**必需字符集合**,`-#` 表示「同时包含 - 和 #」才算命中(#286)。
// 原先两条 char 规则各自做 strings.Contains,是或关系:线上最近一次同步中
// `-` 命中 384、`#` 命中 148,合计 532 条被过滤;改为与关系后只有两者都含的
// 才过滤,其余重新入库。
//
// 空集合永远不命中——否则一条空规则会过滤掉所有商品。
func containsAllRunes(v string, set string) bool {
if set == "" {
return false
}
for _, r := range set {
if !strings.ContainsRune(v, r) {
return false
}
}
return true
}
func (r RuleSet) Match(v string) *models.SYBProductFilter {
if v == "" {
return nil
}
n := sybshop.Normalize(v)
for i, x := range r.Chars {
if strings.Contains(n, x.NormalizedKeyword) {
if containsAllRunes(n, x.NormalizedKeyword) {
return &r.Chars[i]
}
}
@@ -21,25 +21,35 @@ func testDB(t *testing.T) *gorm.DB {
return db
}
func TestMatchORAndNormalization(t *testing.T) {
func TestMatchRequiresEveryStructuralCharacter(t *testing.T) {
r := RuleSet{
Chars: []models.SYBProductFilter{{ID: 1, Kind: "char", NormalizedKeyword: "-"}, {ID: 2, Kind: "char", NormalizedKeyword: "#"}},
Chars: []models.SYBProductFilter{{ID: 1, Kind: "char", NormalizedKeyword: "-#"}},
Keywords: []models.SYBProductFilter{{ID: 3, Kind: "keyword", NormalizedKeyword: "cvc短袖220斤"}},
}
// `[必须]` OR, never AND. Both of these contain "#" but no "-", and on real
// production data that difference is 2118 rows (#269).
// `[必须]` AND, not OR. 结构过滤的 keyword 存的是**必需字符集合**:`-#` 表示
// 同时包含 - 和 # 才过滤(#286)。#269 时是两条规则各自 Contains 的或关系,
// 线上最近一次同步里 `-` 命中 384、`#` 命中 148,合计 532 条被过滤;改成与
// 关系后只剩两者都含的会被过滤,其余重新入库——这是本次改动的全部意义,
// 任何把它退回或关系的改动都必须先推翻这条用例。
if hit := r.Match("新寮-雅伊阁#A057"); hit == nil || hit.ID != 1 {
t.Fatalf("同时含 - 和 # 的应当命中结构过滤,got %v", hit)
}
for _, v := range []string{"DD#004", "300斤牛奶絲圓領#A057"} {
hit := r.Match(v)
if hit == nil || hit.Kind != "char" {
t.Fatalf("%q should match a structural rule, got %v", v, hit)
}
if hit.ID != 2 {
t.Fatalf("%q should be attributed to the \"#\" rule, got id %d", v, hit.ID)
if hit := r.Match(v); hit != nil {
t.Fatalf("%q 只含 #,不应再被结构过滤,got %v", v, hit)
}
}
if hit := r.Match("新寮-雅伊阁大码女装"); hit == nil || hit.ID != 1 {
t.Fatalf("dash-only value should be attributed to the \"-\" rule, got %v", hit)
if hit := r.Match("新寮-雅伊阁大码女装"); hit != nil {
t.Fatalf("只含 - 的不应再被结构过滤,got %v", hit)
}
// 顺序无关:字符集合只要求都出现,不要求相邻或先后。
if hit := r.Match("A#B-C"); hit == nil || hit.ID != 1 {
t.Fatalf("# 在 - 之前也应命中,got %v", hit)
}
// 以下是 #269 定下、本次不变的规则。
if hit := r.Match(""); hit != nil {
t.Fatal("empty variationSku must not match: it is a product without a supplier code")
}
@@ -57,6 +67,35 @@ func TestMatchORAndNormalization(t *testing.T) {
}
}
// 关键词过滤仍然是子串匹配,不受结构过滤改为字符集合的影响。
func TestKeywordFilterStaysSubstringMatching(t *testing.T) {
r := RuleSet{Keywords: []models.SYBProductFilter{{ID: 3, Kind: "keyword", NormalizedKeyword: "印花帽t"}}}
if hit := r.Match("秋冬印花帽T加绒"); hit == nil || hit.Kind != "keyword" {
t.Fatalf("关键词应按子串匹配,got %v", hit)
}
// 字符集合语义若误用到关键词上,下面这个会被判成命中(每个字都出现过)。
if hit := r.Match("印花衬衫 帽子 t恤"); hit != nil {
t.Fatalf("关键词不是字符集合,不应命中,got %v", hit)
}
}
// `[必须]` 全新数据库的种子必须与迁移后的形态一致。migrations.Migrate 每次启动
// 都会 FirstOrCreate 这批种子,若仍播种旧的两条 `-` 和 `#`,就会把迁移
// 1789500000000 合并掉的行重新建回来,过滤静默退回或关系(#286)。
func TestFreshDatabaseSeedsOneStructuralRule(t *testing.T) {
db := testDB(t)
var rows []models.SYBProductFilter
if err := db.Where("kind = ?", "char").Find(&rows).Error; err != nil {
t.Fatal(err)
}
if len(rows) != 1 {
t.Fatalf("结构过滤应当只有一条,实际 %d 条: %+v", len(rows), rows)
}
if rows[0].Keyword != "-#" {
t.Fatalf("结构过滤的必需字符集合应为 -#,实际 %q", rows[0].Keyword)
}
}
// TestUpdateHitsIsPerRule pins the rule that每条规则各记各的命中数. Writing one
// aggregate per kind would make the disable-confirmation dialog quote the same
// figure for "#" and "-" even though they match very different numbers of rows.
@@ -67,8 +106,9 @@ func TestUpdateHitsIsPerRule(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(set.Chars) != 2 {
t.Fatalf("migration should seed 2 structural rules, got %d", len(set.Chars))
// #286:结构过滤合并为一条必需字符集合 `-#`。
if len(set.Chars) != 1 {
t.Fatalf("migration should seed 1 structural rule, got %d", len(set.Chars))
}
byKeyword := map[string]uint64{}
for _, x := range set.Chars {
@@ -0,0 +1,58 @@
package version_local
import (
"runtime"
"go-admin/app/goauto/models"
"go-admin/app/goauto/sybshop"
"go-admin/cmd/migrate/migration"
common "go-admin/common/models"
"gorm.io/gorm"
)
func init() {
_, file, _, _ := runtime.Caller(0)
migration.Migrate.SetVersion(migration.GetFilename(file), migrateSYBCharFilterRequiresBoth)
}
// migrateSYBCharFilterRequiresBoth collapses the two structural filter rules
// into one that only matches when the variationSku contains both characters
// (#286).
//
// `[必须]` #269 seeded two `char` rows (`-` and `#`) and RuleSet.Match treated
// them as OR — a hit on either one skipped the detail. On the last full
// production sync that filtered 532 details (384 by `-`, 148 by `#` without a
// `-`). The rule is now a required-character set: `keyword` holds every
// character that must be present, so `-#` means "both". Everything that
// carries only one of the two starts entering the system again; the user
// confirmed that consequence on 2026-09-15 before this was written.
//
// The old rows are removed rather than disabled: leaving a disabled `-` row
// behind would suggest it can be switched back on, but Match no longer has OR
// semantics for it to return to. Their LastHitCount history is lost with them.
func migrateSYBCharFilterRequiresBoth(db *gorm.DB, version string) error {
return db.Transaction(func(tx *gorm.DB) error {
const required = "-#"
normalized := sybshop.Normalize(required)
// Idempotent: keep the merged row if a previous run already created it,
// then drop every other structural rule.
merged := models.SYBProductFilter{
Kind: "char",
Keyword: required,
NormalizedKeyword: normalized,
Enabled: true,
}
if err := tx.Where("kind = ? AND normalized_keyword = ?", "char", normalized).
FirstOrCreate(&merged).Error; err != nil {
return err
}
if err := tx.Unscoped().
Where("kind = ? AND normalized_keyword <> ?", "char", normalized).
Delete(&models.SYBProductFilter{}).Error; err != nil {
return err
}
return tx.Create(&common.Migration{Version: version}).Error
})
}
@@ -29,9 +29,9 @@
<section>
<h3>结构过滤(不可增删,仅可停用)</h3>
<p class="muted">这两个字符是识别档口商品的判据,命中即不入库。需要变更判据请另行建单评估。</p>
<p class="muted">variationSku 需要<strong>同时包含</strong>下方列出的每一个字符,才会被当成档口商品不入库;只包含其中一个的会正常入库。需要变更判据请另行建单评估。</p>
<el-table v-loading="loading" :data="chars" row-key="id" border stripe>
<el-table-column label="字符" prop="keyword" width="180" />
<el-table-column label="必需字符" prop="keyword" width="180" />
<el-table-column label="状态" width="150">
<template #default="{ row }">
<el-switch
@@ -210,7 +210,7 @@ export default {
async confirmDisableChar(row) {
const scale = row.lastHitCount == null ? '尚不清楚有多少' : `约 ${row.lastHitCount} 条`
await ElMessageBox.confirm(
`停用「${row.keyword}」后,SYB 同步将不再过滤含 ${row.keyword} 的档口商品。按最近一次同步数据估算,${scale}明细会恢复入库。`,
`停用后,SYB 同步将不再过滤任何档口商品(结构过滤只有这一条)。按最近一次同步数据估算,${scale}明细会恢复入库。`,
`确认停用结构过滤「${row.keyword}」?`,
{ type: 'warning', confirmButtonText: '确认停用', cancelButtonText: '取消', confirmButtonClass: 'el-button--danger' }
)