feat(yeeke): show missing-marked and recovered counts per sync run (#338)
Persist how many return items each sync run flipped to "missing" and how many came back to "ok" (yeeke_sync_run.missing_marked_count / recovered_count, migration 1789801000000), return them from the sync-runs API and add 「标记不可用」「恢复可用」 columns to the sync-runs page. When the 20% safety valve skips marking the count stays 0 and the reason remains in error_message. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NTDbDcwbDw1TSAcE6wfh2F
This commit is contained in:
@@ -83,6 +83,9 @@ type YeekeSyncRun struct {
|
|||||||
UpdatedCount int `gorm:"not null;default:0"`
|
UpdatedCount int `gorm:"not null;default:0"`
|
||||||
SkippedCount int `gorm:"not null;default:0"`
|
SkippedCount int `gorm:"not null;default:0"`
|
||||||
FailedCount int `gorm:"not null;default:0"`
|
FailedCount int `gorm:"not null;default:0"`
|
||||||
|
// #338: items flipped to "missing" / back to "ok" by this run.
|
||||||
|
MissingMarkedCount int `gorm:"not null;default:0"`
|
||||||
|
RecoveredCount int `gorm:"not null;default:0"`
|
||||||
ErrorMessage string `gorm:"size:1000;not null;default:''"`
|
ErrorMessage string `gorm:"size:1000;not null;default:''"`
|
||||||
StartedAt time.Time `gorm:"not null"`
|
StartedAt time.Time `gorm:"not null"`
|
||||||
FinishedAt *time.Time
|
FinishedAt *time.Time
|
||||||
|
|||||||
@@ -52,6 +52,8 @@ type SyncRunDTO struct {
|
|||||||
UpdatedCount int `json:"updatedCount"`
|
UpdatedCount int `json:"updatedCount"`
|
||||||
SkippedCount int `json:"skippedCount"`
|
SkippedCount int `json:"skippedCount"`
|
||||||
FailedCount int `json:"failedCount"`
|
FailedCount int `json:"failedCount"`
|
||||||
|
MissingMarkedCount int `json:"missingMarkedCount"`
|
||||||
|
RecoveredCount int `json:"recoveredCount"`
|
||||||
ErrorMessage string `json:"errorMessage"`
|
ErrorMessage string `json:"errorMessage"`
|
||||||
StartedAt string `json:"startedAt"`
|
StartedAt string `json:"startedAt"`
|
||||||
FinishedAt *string `json:"finishedAt"`
|
FinishedAt *string `json:"finishedAt"`
|
||||||
@@ -62,7 +64,8 @@ func toDTO(r models.YeekeSyncRun) SyncRunDTO {
|
|||||||
dto := SyncRunDTO{
|
dto := SyncRunDTO{
|
||||||
ID: r.ID, Status: r.Status, Trigger: r.Trigger, TotalPages: r.TotalPages,
|
ID: r.ID, Status: r.Status, Trigger: r.Trigger, TotalPages: r.TotalPages,
|
||||||
ReadCount: r.ReadCount, CreatedCount: r.CreatedCount, UpdatedCount: r.UpdatedCount,
|
ReadCount: r.ReadCount, CreatedCount: r.CreatedCount, UpdatedCount: r.UpdatedCount,
|
||||||
SkippedCount: r.SkippedCount, FailedCount: r.FailedCount, ErrorMessage: r.ErrorMessage,
|
SkippedCount: r.SkippedCount, FailedCount: r.FailedCount,
|
||||||
|
MissingMarkedCount: r.MissingMarkedCount, RecoveredCount: r.RecoveredCount, ErrorMessage: r.ErrorMessage,
|
||||||
StartedAt: r.StartedAt.UTC().Format("2006-01-02T15:04:05Z"),
|
StartedAt: r.StartedAt.UTC().Format("2006-01-02T15:04:05Z"),
|
||||||
}
|
}
|
||||||
if r.FinishedAt != nil {
|
if r.FinishedAt != nil {
|
||||||
|
|||||||
@@ -43,10 +43,14 @@ type Report struct {
|
|||||||
// MissingMarked is the number of yeeke_return_item rows flipped from
|
// MissingMarked is the number of yeeke_return_item rows flipped from
|
||||||
// "ok" to "missing" by this run's completion (#338). It is only ever
|
// "ok" to "missing" by this run's completion (#338). It is only ever
|
||||||
// non-zero on a run whose page walk finished naturally, wrote no
|
// non-zero on a run whose page walk finished naturally, wrote no
|
||||||
// failures, and stayed under the 20% safety-valve threshold; it is not
|
// failures, and stayed under the 20% safety-valve threshold. Persisted
|
||||||
// persisted as a run column, only reported here and folded into
|
// as yeeke_sync_run.missing_marked_count; when the safety valve skips
|
||||||
// error_message when the safety valve skips marking.
|
// marking it stays 0 and the reason goes to error_message.
|
||||||
MissingMarked int
|
MissingMarked int
|
||||||
|
// Recovered is the number of yeeke_return_item rows that were "missing"
|
||||||
|
// before this run and reappeared in it, flipping back to "ok" (#338).
|
||||||
|
// Persisted as yeeke_sync_run.recovered_count.
|
||||||
|
Recovered int
|
||||||
}
|
}
|
||||||
|
|
||||||
// knownClaimStatuses lists the status values the sync code currently
|
// knownClaimStatuses lists the status values the sync code currently
|
||||||
@@ -184,7 +188,7 @@ func (s *Service) run(ctx context.Context, r *models.YeekeSyncRun) (Report, erro
|
|||||||
var runErr error
|
var runErr error
|
||||||
defer func() {
|
defer func() {
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
updates := map[string]any{"status": rep.Status, "total_pages": rep.TotalPages, "read_count": rep.Read, "created_count": rep.Created, "updated_count": rep.Updated, "skipped_count": rep.Skipped, "failed_count": rep.Failed, "error_message": errMsg, "active_slot": nil, "lease_owner": "", "lease_expires_at": nil, "finished_at": now}
|
updates := map[string]any{"status": rep.Status, "total_pages": rep.TotalPages, "read_count": rep.Read, "created_count": rep.Created, "updated_count": rep.Updated, "skipped_count": rep.Skipped, "failed_count": rep.Failed, "missing_marked_count": rep.MissingMarked, "recovered_count": rep.Recovered, "error_message": errMsg, "active_slot": nil, "lease_owner": "", "lease_expires_at": nil, "finished_at": now}
|
||||||
if rep.Status == "succeeded" {
|
if rep.Status == "succeeded" {
|
||||||
updates["last_success_at"] = now
|
updates["last_success_at"] = now
|
||||||
}
|
}
|
||||||
@@ -235,7 +239,7 @@ func (s *Service) run(ctx context.Context, r *models.YeekeSyncRun) (Report, erro
|
|||||||
}
|
}
|
||||||
seen[finger] = true
|
seen[finger] = true
|
||||||
for _, x := range p.Records {
|
for _, x := range p.Records {
|
||||||
created, updated, err := s.upsert(ctx, x)
|
created, updated, recovered, err := s.upsert(ctx, x)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rep.Failed++
|
rep.Failed++
|
||||||
if firstWriteErr == nil {
|
if firstWriteErr == nil {
|
||||||
@@ -244,6 +248,7 @@ func (s *Service) run(ctx context.Context, r *models.YeekeSyncRun) (Report, erro
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
rep.Read++
|
rep.Read++
|
||||||
|
rep.Recovered += recovered
|
||||||
if created {
|
if created {
|
||||||
rep.Created++
|
rep.Created++
|
||||||
} else if updated {
|
} else if updated {
|
||||||
@@ -354,7 +359,10 @@ func pageFingerprint(p yeekeclient.ReturnPage) string {
|
|||||||
h := sha256.Sum256(b)
|
h := sha256.Sum256(b)
|
||||||
return hex.EncodeToString(h[:])
|
return hex.EncodeToString(h[:])
|
||||||
}
|
}
|
||||||
func (s *Service) upsert(ctx context.Context, p yeekeclient.ReturnPackage) (bool, bool, error) {
|
|
||||||
|
// upsert writes one package and its items. recovered counts items that were
|
||||||
|
// marked "missing" before and reappeared in this run (#338).
|
||||||
|
func (s *Service) upsert(ctx context.Context, p yeekeclient.ReturnPackage) (created bool, updated bool, recovered int, err error) {
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
key := packageKey(p)
|
key := packageKey(p)
|
||||||
status := external(p.Status)
|
status := external(p.Status)
|
||||||
@@ -363,7 +371,7 @@ func (s *Service) upsert(ctx context.Context, p yeekeclient.ReturnPackage) (bool
|
|||||||
// behind without its items. Rows are always inserted fully populated —
|
// behind without its items. Rows are always inserted fully populated —
|
||||||
// MySQL strict mode (NO_ZERO_DATE) rejects the zero last_synced_at an
|
// MySQL strict mode (NO_ZERO_DATE) rejects the zero last_synced_at an
|
||||||
// empty placeholder insert would carry, which failed every record.
|
// empty placeholder insert would carry, which failed every record.
|
||||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
err = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||||
var row models.YeekeReturnPackage
|
var row models.YeekeReturnPackage
|
||||||
e := tx.Where("external_id = ?", key).First(&row).Error
|
e := tx.Where("external_id = ?", key).First(&row).Error
|
||||||
isNew = errors.Is(e, gorm.ErrRecordNotFound)
|
isNew = errors.Is(e, gorm.ErrRecordNotFound)
|
||||||
@@ -411,6 +419,9 @@ func (s *Service) upsert(ctx context.Context, p yeekeclient.ReturnPackage) (bool
|
|||||||
case ie != nil:
|
case ie != nil:
|
||||||
return ie
|
return ie
|
||||||
default:
|
default:
|
||||||
|
if existing.SyncStatus == "missing" {
|
||||||
|
recovered++
|
||||||
|
}
|
||||||
iv := map[string]any{"item_id": item.ItemID, "variation_id": item.VariationID, "item_name": item.ItemName, "variation_name": item.VariationName, "image": item.Image, "quantity": item.Quantity, "last_synced_at": now, "sync_status": "ok", "missing_since": (*time.Time)(nil)}
|
iv := map[string]any{"item_id": item.ItemID, "variation_id": item.VariationID, "item_name": item.ItemName, "variation_name": item.VariationName, "image": item.Image, "quantity": item.Quantity, "last_synced_at": now, "sync_status": "ok", "missing_since": (*time.Time)(nil)}
|
||||||
if e = tx.Model(&existing).Updates(iv).Error; e != nil {
|
if e = tx.Model(&existing).Updates(iv).Error; e != nil {
|
||||||
return e
|
return e
|
||||||
@@ -420,7 +431,7 @@ func (s *Service) upsert(ctx context.Context, p yeekeclient.ReturnPackage) (bool
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, false, err
|
return false, false, 0, err
|
||||||
}
|
}
|
||||||
return isNew, !isNew, nil
|
return isNew, !isNew, recovered, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -366,9 +366,24 @@ func TestReappearingRecordRecoversFromMissing(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv3.Close()
|
defer srv3.Close()
|
||||||
s.client, _ = yeekeclient.New(srv3.URL)
|
s.client, _ = yeekeclient.New(srv3.URL)
|
||||||
if _, err := s.Sync(context.Background(), "manual"); err != nil {
|
rep3, err := s.Sync(context.Background(), "manual")
|
||||||
|
if err != nil {
|
||||||
t.Fatalf("run 3 (recovery): %v", err)
|
t.Fatalf("run 3 (recovery): %v", err)
|
||||||
}
|
}
|
||||||
|
if rep3.Recovered != 1 || rep3.MissingMarked != 0 {
|
||||||
|
t.Fatalf("run3 Recovered=%d MissingMarked=%d, want 1 and 0", rep3.Recovered, rep3.MissingMarked)
|
||||||
|
}
|
||||||
|
// Both counters are persisted on the run rows shown by the sync-runs page.
|
||||||
|
var run2, run3 models.YeekeSyncRun
|
||||||
|
if e := db.First(&run2, rep2.RunID).Error; e != nil {
|
||||||
|
t.Fatal(e)
|
||||||
|
}
|
||||||
|
if e := db.First(&run3, rep3.RunID).Error; e != nil {
|
||||||
|
t.Fatal(e)
|
||||||
|
}
|
||||||
|
if run2.MissingMarkedCount != 1 || run2.RecoveredCount != 0 || run3.MissingMarkedCount != 0 || run3.RecoveredCount != 1 {
|
||||||
|
t.Fatalf("persisted counts run2=(%d,%d) run3=(%d,%d), want (1,0) and (0,1)", run2.MissingMarkedCount, run2.RecoveredCount, run3.MissingMarkedCount, run3.RecoveredCount)
|
||||||
|
}
|
||||||
|
|
||||||
// A fresh variable is used here (not the p1 declared above): GORM's Scan
|
// A fresh variable is used here (not the p1 declared above): GORM's Scan
|
||||||
// does not reset an already non-nil pointer field to nil when the new
|
// does not reset an already non-nil pointer field to nil when the new
|
||||||
|
|||||||
+25
@@ -0,0 +1,25 @@
|
|||||||
|
package version_local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go-admin/app/goauto/migrations"
|
||||||
|
"go-admin/cmd/migrate/migration"
|
||||||
|
common "go-admin/common/models"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"runtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
// #338: adds yeeke_sync_run.missing_marked_count and recovered_count (registered in
|
||||||
|
// migrations.MigratedModels) on databases whose earlier versions are already
|
||||||
|
// recorded in sys_migration.
|
||||||
|
func init() {
|
||||||
|
_, f, _, _ := runtime.Caller(0)
|
||||||
|
migration.Migrate.SetVersion(migration.GetFilename(f), migrateYeekeSyncRunMissingCounts)
|
||||||
|
}
|
||||||
|
func migrateYeekeSyncRunMissingCounts(db *gorm.DB, version string) error {
|
||||||
|
return db.Transaction(func(tx *gorm.DB) error {
|
||||||
|
if err := migrations.Migrate(tx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return tx.Create(&common.Migration{Version: version}).Error
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -41,6 +41,8 @@
|
|||||||
<el-table-column label="更新" prop="updatedCount" width="70" />
|
<el-table-column label="更新" prop="updatedCount" width="70" />
|
||||||
<el-table-column label="跳过" prop="skippedCount" width="70" />
|
<el-table-column label="跳过" prop="skippedCount" width="70" />
|
||||||
<el-table-column label="失败" prop="failedCount" width="70" />
|
<el-table-column label="失败" prop="failedCount" width="70" />
|
||||||
|
<el-table-column label="标记不可用" prop="missingMarkedCount" width="96" />
|
||||||
|
<el-table-column label="恢复可用" prop="recoveredCount" width="84" />
|
||||||
<el-table-column label="脱敏原因" min-width="200"><template #default="{ row }">{{ row.errorMessage || '—' }}</template></el-table-column>
|
<el-table-column label="脱敏原因" min-width="200"><template #default="{ row }">{{ row.errorMessage || '—' }}</template></el-table-column>
|
||||||
</el-table>
|
</el-table>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
Reference in New Issue
Block a user