diff --git a/Sense/server/cmd/migrate/migration/version/2026081414000_device.go b/Sense/server/cmd/migrate/migration/version/2026081414000_device.go index 7eae8e8..96d8553 100644 --- a/Sense/server/cmd/migrate/migration/version/2026081414000_device.go +++ b/Sense/server/cmd/migrate/migration/version/2026081414000_device.go @@ -1,8 +1,11 @@ package version import ( + "database/sql" + "encoding/json" "fmt" "runtime" + "strings" "gorm.io/gorm" "gorm.io/gorm/clause" @@ -34,6 +37,9 @@ func init() { func migrateSenseDeviceLedger(db *gorm.DB, version string) error { return db.Transaction(func(tx *gorm.DB) error { + if err := prepareLegacyDeviceCapabilities(tx); err != nil { + return err + } if err := tx.AutoMigrate(&deviceModels.Device{}, &credential.DeviceCredential{}, &deviceCasbinRule{}); err != nil { return err } @@ -89,6 +95,104 @@ func migrateSenseDeviceLedger(db *gorm.DB, version string) error { }) } +var supportedLegacyCapabilities = map[string]struct{}{ + "video": {}, "radar": {}, "contact": {}, "button": {}, "wearable": {}, "other": {}, +} + +type legacyDeviceCapabilitiesRow struct { + ID string + Capabilities sql.NullString +} + +// prepareLegacyDeviceCapabilities upgrades the pre-GoAdmin text column before +// GORM sees it. PostgreSQL cannot cast the old empty-string default to jsonb, +// and old rows stored a single capability token rather than a JSON array. +func prepareLegacyDeviceCapabilities(tx *gorm.DB) error { + if tx.Dialector.Name() != "postgres" { + return nil + } + type columnMetadata struct { + DataType string + } + var column columnMetadata + result := tx.Raw(`SELECT data_type + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'sense_devices' + AND column_name = 'capabilities'`).Scan(&column) + if result.Error != nil { + return fmt.Errorf("inspect sense_devices.capabilities: %w", result.Error) + } + if result.RowsAffected == 0 || column.DataType == "json" || column.DataType == "jsonb" { + return nil + } + if column.DataType != "text" && column.DataType != "character varying" { + return fmt.Errorf("sense_devices.capabilities has unsupported legacy type %q", column.DataType) + } + if err := tx.Exec(`LOCK TABLE "sense_devices" IN ACCESS EXCLUSIVE MODE`).Error; err != nil { + return fmt.Errorf("lock sense_devices for capabilities migration: %w", err) + } + var rows []legacyDeviceCapabilitiesRow + if err := tx.Raw(`SELECT id, capabilities FROM "sense_devices" ORDER BY id`).Scan(&rows).Error; err != nil { + return fmt.Errorf("read legacy device capabilities: %w", err) + } + canonical := make(map[string]string, len(rows)) + invalid := 0 + for _, row := range rows { + value, err := canonicalLegacyCapabilities(row.Capabilities) + if err != nil { + invalid++ + continue + } + canonical[row.ID] = value + } + if invalid > 0 { + return fmt.Errorf("sense_devices.capabilities contains unsupported legacy data in %d row(s); migration rolled back", invalid) + } + if err := tx.Exec(`ALTER TABLE "sense_devices" ALTER COLUMN "capabilities" DROP DEFAULT`).Error; err != nil { + return fmt.Errorf("drop legacy capabilities default: %w", err) + } + for _, row := range rows { + if err := tx.Exec(`UPDATE "sense_devices" SET "capabilities" = ? WHERE "id" = ?`, canonical[row.ID], row.ID).Error; err != nil { + return fmt.Errorf("normalize legacy device capabilities: %w", err) + } + } + if err := tx.Exec(`ALTER TABLE "sense_devices" ALTER COLUMN "capabilities" TYPE jsonb USING "capabilities"::jsonb`).Error; err != nil { + return fmt.Errorf("convert capabilities to jsonb: %w", err) + } + if err := tx.Exec(`ALTER TABLE "sense_devices" ALTER COLUMN "capabilities" SET DEFAULT '[]'::jsonb`).Error; err != nil { + return fmt.Errorf("set jsonb capabilities default: %w", err) + } + return nil +} + +func canonicalLegacyCapabilities(value sql.NullString) (string, error) { + if !value.Valid || strings.TrimSpace(value.String) == "" { + return "[]", nil + } + trimmed := strings.TrimSpace(value.String) + if _, ok := supportedLegacyCapabilities[trimmed]; ok { + encoded, _ := json.Marshal([]string{trimmed}) + return string(encoded), nil + } + if !strings.HasPrefix(trimmed, "[") { + return "", fmt.Errorf("legacy capability value is not an array") + } + var values []string + if err := json.Unmarshal([]byte(trimmed), &values); err != nil || values == nil || len(values) > 16 { + return "", fmt.Errorf("legacy capability array is invalid") + } + for index, item := range values { + item = strings.TrimSpace(item) + if _, ok := supportedLegacyCapabilities[item]; !ok { + return "", fmt.Errorf("legacy capability array contains an unsupported value") + } + values[index] = item + } + encoded, _ := json.Marshal(values) + return string(encoded), nil +} + func ensureDeviceMenu(tx *gorm.DB, desired migrationModels.SysMenu) (migrationModels.SysMenu, error) { var menu migrationModels.SysMenu err := tx.Where("menu_name = ?", desired.MenuName).First(&menu).Error diff --git a/Sense/server/cmd/migrate/migration/version/2026081414000_device_test.go b/Sense/server/cmd/migrate/migration/version/2026081414000_device_test.go new file mode 100644 index 0000000..b8cf7a3 --- /dev/null +++ b/Sense/server/cmd/migrate/migration/version/2026081414000_device_test.go @@ -0,0 +1,207 @@ +package version + +import ( + "database/sql" + "os" + "strings" + "testing" + + "gorm.io/driver/postgres" + "gorm.io/gorm" + + deviceModels "git.ilapage.cn/ila/yovision/Sense/server/app/sense/device/models" + migrationModels "git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration/models" + common "git.ilapage.cn/ila/yovision/Sense/server/common/models" +) + +func TestCanonicalLegacyCapabilities(t *testing.T) { + tests := []struct { + name string + input sql.NullString + want string + wantErr bool + }{ + {name: "null", input: sql.NullString{}, want: "[]"}, + {name: "blank", input: sql.NullString{String: " ", Valid: true}, want: "[]"}, + {name: "single", input: sql.NullString{String: " video ", Valid: true}, want: `["video"]`}, + {name: "array", input: sql.NullString{String: `["radar", "contact"]`, Valid: true}, want: `["radar","contact"]`}, + {name: "empty array", input: sql.NullString{String: `[]`, Valid: true}, want: `[]`}, + {name: "unknown", input: sql.NullString{String: "unknown", Valid: true}, wantErr: true}, + {name: "object", input: sql.NullString{String: `{"video":true}`, Valid: true}, wantErr: true}, + {name: "unknown array item", input: sql.NullString{String: `["video","unknown"]`, Valid: true}, wantErr: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := canonicalLegacyCapabilities(test.input) + if test.wantErr { + if err == nil { + t.Fatalf("expected error, got %q", got) + } + return + } + if err != nil || got != test.want { + t.Fatalf("got %q, %v; want %q", got, err, test.want) + } + }) + } +} + +func TestDeviceCapabilitiesMigrationOnPostgres(t *testing.T) { + dsn := os.Getenv("SENSE_DEVICE_MIGRATION_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("set SENSE_DEVICE_MIGRATION_TEST_DATABASE_URL to run the PostgreSQL migration test") + } + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + const schema = "sense_device_92_test" + if err = db.Exec("DROP SCHEMA IF EXISTS " + schema + " CASCADE").Error; err != nil { + t.Fatal(err) + } + if err = db.Exec("CREATE SCHEMA " + schema).Error; err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Exec("DROP SCHEMA IF EXISTS " + schema + " CASCADE") }) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetMaxOpenConns(1) + if err = db.Exec("SET search_path TO " + schema).Error; err != nil { + t.Fatal(err) + } + + t.Run("converts legacy values and remains idempotent", func(t *testing.T) { + resetLegacyDeviceTable(t, db) + for _, row := range [][2]string{{"single", "video"}, {"blank", ""}, {"array", `["radar","contact"]`}} { + if err = db.Exec(`INSERT INTO sense_devices (id, capabilities) VALUES (?, ?)`, row[0], row[1]).Error; err != nil { + t.Fatal(err) + } + } + if err = db.Transaction(prepareLegacyDeviceCapabilities); err != nil { + t.Fatal(err) + } + if err = db.AutoMigrate(&deviceModels.Device{}); err != nil { + t.Fatalf("AutoMigrate after compatibility conversion: %v", err) + } + assertCapabilitiesColumn(t, db, "jsonb", "'[]'::jsonb") + var values []struct{ ID, Capabilities string } + if err = db.Raw(`SELECT id, capabilities::text AS capabilities FROM sense_devices ORDER BY id`).Scan(&values).Error; err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, value := range values { + got[value.ID] = strings.ReplaceAll(value.Capabilities, " ", "") + } + if got["single"] != `["video"]` || got["blank"] != `[]` || got["array"] != `["radar","contact"]` { + t.Fatalf("unexpected converted values: %#v", got) + } + if err = db.Transaction(prepareLegacyDeviceCapabilities); err != nil { + t.Fatalf("repeat migration: %v", err) + } + }) + + t.Run("rejects unknown values without partial conversion", func(t *testing.T) { + resetLegacyDeviceTable(t, db) + if err = db.Exec(`INSERT INTO sense_devices (id, capabilities) VALUES ('valid', 'video'), ('invalid', 'unknown')`).Error; err != nil { + t.Fatal(err) + } + if err = db.Transaction(prepareLegacyDeviceCapabilities); err == nil { + t.Fatal("expected unsupported legacy data error") + } + assertCapabilitiesColumn(t, db, "text", "''::text") + var value string + if err = db.Raw(`SELECT capabilities FROM sense_devices WHERE id = 'valid'`).Scan(&value).Error; err != nil { + t.Fatal(err) + } + if value != "video" { + t.Fatalf("transaction left partial data: %q", value) + } + }) + + t.Run("does nothing when the table is absent", func(t *testing.T) { + if err = db.Exec(`DROP TABLE sense_devices`).Error; err != nil { + t.Fatal(err) + } + if err = db.Transaction(prepareLegacyDeviceCapabilities); err != nil { + t.Fatal(err) + } + }) + + t.Run("completes the device migration and records its version", func(t *testing.T) { + resetLegacyDeviceTable(t, db) + if err = db.Exec(`INSERT INTO sense_devices (id, capabilities) VALUES ('legacy', 'video')`).Error; err != nil { + t.Fatal(err) + } + if err = db.AutoMigrate(&migrationModels.SysRole{}, &migrationModels.SysMenu{}, &deviceCasbinRule{}, &common.Migration{}); err != nil { + t.Fatal(err) + } + for _, role := range []string{"implementation_operator", "site_admin", "viewer"} { + if err = db.Create(&migrationModels.SysRole{RoleName: role, RoleKey: role, Status: "2"}).Error; err != nil { + t.Fatal(err) + } + } + const migrationVersion = "2026081414000_device.go" + if err = migrateSenseDeviceLedger(db, migrationVersion); err != nil { + t.Fatal(err) + } + assertCapabilitiesColumn(t, db, "jsonb", "'[]'::jsonb") + var migratedValue string + if err = db.Raw(`SELECT capabilities::text FROM sense_devices WHERE id = 'legacy'`).Scan(&migratedValue).Error; err != nil { + t.Fatal(err) + } + if strings.ReplaceAll(migratedValue, " ", "") != `["video"]` { + t.Fatalf("unexpected full-migration value: %q", migratedValue) + } + var applied int64 + if err = db.Model(&common.Migration{}).Where("version = ?", migrationVersion).Count(&applied).Error; err != nil { + t.Fatal(err) + } + if applied != 1 { + t.Fatalf("migration record count=%d", applied) + } + }) +} + +func resetLegacyDeviceTable(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec(`DROP TABLE IF EXISTS sense_devices`).Error; err != nil { + t.Fatal(err) + } + if err := db.Exec(`CREATE TABLE sense_devices ( + id text PRIMARY KEY, + name varchar(128) NOT NULL DEFAULT '', + location varchar(255) NOT NULL DEFAULT '', + modality varchar(32) NOT NULL DEFAULT 'video', + capabilities text NOT NULL DEFAULT '', + status varchar(32) NOT NULL DEFAULT 'active', + adapter_status varchar(32) NOT NULL DEFAULT 'ready', + rtsp_credential_same_as_onvif boolean NOT NULL DEFAULT true, + credential_updated_at timestamptz, + retry_requested_at timestamptz, + version bigint NOT NULL DEFAULT 1, + create_by bigint NOT NULL DEFAULT 0, + update_by bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + deleted_at timestamptz + )`).Error; err != nil { + t.Fatal(err) + } +} + +func assertCapabilitiesColumn(t *testing.T, db *gorm.DB, wantType, wantDefault string) { + t.Helper() + var column struct{ DataType, ColumnDefault string } + if err := db.Raw(`SELECT data_type, column_default + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'sense_devices' + AND column_name = 'capabilities'`).Scan(&column).Error; err != nil { + t.Fatal(err) + } + if column.DataType != wantType || column.ColumnDefault != wantDefault { + t.Fatalf("type/default=%q/%q; want %q/%q", column.DataType, column.ColumnDefault, wantType, wantDefault) + } +} diff --git a/docs/06-troubleshooting.md b/docs/06-troubleshooting.md index 130cb14..49db000 100644 --- a/docs/06-troubleshooting.md +++ b/docs/06-troubleshooting.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件) wiki_page: Troubleshooting wiki_url: https://git.ilapage.cn/ila/yovision/wiki/Troubleshooting -wiki_revision: 3523b7cd1f126ebc3ef77414cb6d658103f2491d -synchronized_at: 2026-08-15T01:13:11Z +wiki_revision: a169b2323323d9de6304e9b430ddbe9888ea1d25 +synchronized_at: 2026-08-15T07:16:25Z # 故障排查 @@ -116,3 +116,21 @@ synchronized_at: 2026-08-15T01:13:11Z | viewer 看得到页面但不能保存 | 符合只读权限;由 implementation_operator、site_admin 或 admin 完成配置。 | | 键盘无法操作顶点 | Tab 聚焦画布或编号顶点;Enter/Space 添加中心点,方向键移动,Delete/Backspace 删除。检查浏览器焦点轮廓是否可见。 | + + +## Sense 旧设备能力字段迁移排错 + +启动迁移出现 `字段 "capabilities" 的默认值不能转换成类型 jsonb (SQLSTATE 42804)`,表示数据库仍保留旧版 `sense_devices.capabilities text DEFAULT ''`,而当前 GoAdmin 派生模型要求 JSONB。不要跳过迁移、删除设备记录或只手工删除默认值;旧单值数据仍可能在下一步转换失败。 + +工单 #92 的兼容迁移会在同一事务内锁定设备表并先验证全部旧值:空值转为 `[]`,`video`、`radar`、`contact`、`button`、`wearable`、`other` 等旧单值转为 JSON 数组,合法 JSON 数组保持数组。未知值或非数组 JSON 会拒绝迁移并整体回滚,不输出具体业务值。 + +处理步骤: + +1. 停止所有连接该 Sense 数据库的服务实例。 +2. 使用 `backup-sense.bat` 创建 PostgreSQL custom-format 备份,并确认备份文件可读取。 +3. 部署包含 #92 的新 `sense.exe` 后重新运行 `migrate-sense.bat` 或正常启动。 +4. 若提示“unsupported legacy data”,不要直接改表;保留错误、恢复测试副本并由维护人员确认旧能力语义。 +5. 迁移成功后确认设备仍存在、能力标签正确,再启动其他实例。 + +正式数据库未备份时不得执行该结构迁移。需要回退版本时停止服务并从迁移前备份恢复,不把 JSONB 反向猜测为旧文本。 + diff --git a/docs/task/92-Sense旧设备能力JSONB兼容迁移.md b/docs/task/92-Sense旧设备能力JSONB兼容迁移.md new file mode 100644 index 0000000..453fad3 --- /dev/null +++ b/docs/task/92-Sense旧设备能力JSONB兼容迁移.md @@ -0,0 +1,74 @@ + +generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件) +wiki_page: Task-92-Sense旧设备能力JSONB兼容迁移 +wiki_url: https://git.ilapage.cn/ila/yovision/wiki/Task-92-Sense%E6%97%A7%E8%AE%BE%E5%A4%87%E8%83%BD%E5%8A%9BJSONB%E5%85%BC%E5%AE%B9%E8%BF%81%E7%A7%BB.- +wiki_revision: d10479f732597bd90dac4de03edc0cd4b39b6352 +synchronized_at: 2026-08-15T07:19:37Z + + +# 92 Sense旧设备能力JSONB兼容迁移 + +- 类型:缺陷 +- 所属 Epic:#7 +- 所属 MVP / 版本:#8 +- 状态:待验收 +- 日期:2026-08-15 +- Gitea 工单:https://git.ilapage.cn/ila/yovision/issues/92 +- Wiki 页面:Task-92-Sense旧设备能力JSONB兼容迁移 +- Wiki revision:见本地镜像头 + +## 背景与目标 + +用户通过 Windows 包启动 Sense 时,设备迁移报 PostgreSQL `SQLSTATE 42804`。只读诊断确认旧数据库的 `sense_devices.capabilities` 为 `text NOT NULL DEFAULT ''::text`,一条旧记录保存受支持的单值能力但不是 JSON;新模型要求 JSONB,GORM AutoMigrate 无法直接转换旧默认值和数据。 + +目标是在不删除设备、不跳过迁移、不修改当前用户数据库的前提下,为旧 schema 提供确定性、可回滚的 JSONB 兼容迁移。 + +## 最终方案 + +在现有 `2026081414000` 设备迁移事务开头执行 PostgreSQL 专用兼容步骤。受影响数据库尚未登记该迁移版本,后置新版本无法越过失败点,因此兼容逻辑必须放在原失败迁移内。 + +兼容步骤只处理既有 text/varchar 列:取得 ACCESS EXCLUSIVE 表锁,读取并验证全部旧值后才开始改变默认值和数据。空值转为 `[]`;六种受支持旧单值转为单元素 JSON 数组;合法字符串数组规范化后保持语义。未知单值、对象、非字符串数组、未知数组元素和超过 16 项的数组会返回不含业务值的错误,整个事务回滚。 + +全部验证通过后删除旧 text 默认值,参数化更新规范 JSON,使用显式 `USING capabilities::jsonb` 转型并设置 `'[]'::jsonb` 默认值,再继续原有 GORM AutoMigrate、菜单、权限和迁移版本登记。非 PostgreSQL、表/列不存在以及已是 json/jsonb 时不执行旧值转换。 + +## 修改文件 + +- `Sense/server/cmd/migrate/migration/version/2026081414000_device.go`:事务内旧 text 能力验证、转换、锁表和 JSONB 默认值处理。 +- `Sense/server/cmd/migrate/migration/version/2026081414000_device_test.go`:纯函数和隔离 PostgreSQL 17 回归,包括完整设备迁移与版本登记。 +- Wiki `Troubleshooting`、`docs/06-troubleshooting.md`:备份、重试、未知旧值和回退说明。 +- `wiki-docs.json`、`docs/task/92-Sense旧设备能力JSONB兼容迁移.md`:任务归档登记和镜像。 + +## 验收结果 + +| 验收标准 | 结果 | +|---|---| +| 旧 text 默认值不再触发 SQLSTATE 42804 | 通过;隔离 PostgreSQL 重现结构成功转为 jsonb | +| 旧单值无损转换为 JSONB 数组 | 通过;`video` 转为单元素数组 | +| 空值与合法数组正确处理 | 通过;空字符串转空数组,合法数组保持顺序与值 | +| 未知值拒绝且无半成品 | 通过;类型、默认值和已验证行均保持旧状态 | +| 新库、jsonb 和重复执行兼容 | 通过;无表跳过、jsonb 重复调用无操作 | +| 全量 Go 和迁移回归通过 | 通过 | +| 当前用户数据库未被修改 | 通过;仅执行只读诊断,写测试使用独立数据库并在结束后删除 | +| Wiki 与归档一致 | 受影响页面定向检查通过 | + +## 测试 + +- `go test ./cmd/migrate/migration/version -run TestCanonicalLegacyCapabilities -count=1 -v`:通过。 +- 隔离 PostgreSQL 17 `TestDeviceCapabilitiesMigrationOnPostgres`:旧单值/空值/合法数组、未知值事务回滚、无表、重复调用、完整设备迁移和 `sys_migration` 登记全部通过。 +- 隔离数据库 `sense92_migration_test` 测试前确认不存在,测试后确认计数为 0。 +- `go test ./...`、`go vet ./...`、`go build ./...`:通过。 +- `go test -race ./cmd/migrate/migration/version -run TestCanonicalLegacyCapabilities -count=1`:通过。 +- `git diff --check`:通过。 +- 受影响 Wiki 定向同步与检查:通过。 +- **未验证部分**:按工单安全边界未在当前用户 `sense` 数据库执行写迁移,也未替换当前 `Sense/dist` 中的待验收 #70 二进制;需先备份,再部署包含 #92 的新包进行最终启动验收。全量 Wiki 检查仍会先发现待验收 PR #89 的 #70 镜像尚未合入 `dev`。 + +## 遗留问题 + +- 当前交付包仍含 #92 修复前的 `sense.exe`。#92 合入 `dev` 后需让 #70 交付分支吸收该提交并重新打包,用户备份数据库后再运行迁移。 +- Harness strict 的 #66/#67 既有归档格式问题不属于本工单。 + +## 相关提交 + +- `68b436a` 兼容旧设备能力字段迁移。 +- `fe3badf` 覆盖旧设备完整迁移链。 +- `aee5f45` 记录设备能力迁移排错。 diff --git a/wiki-docs.json b/wiki-docs.json index 9eab71e..cf70f4f 100644 --- a/wiki-docs.json +++ b/wiki-docs.json @@ -112,6 +112,10 @@ "page": "Task-65-Sense设备台账与凭据边界", "path": "docs/task/65-Sense设备台账与凭据边界.md" }, + { + "page": "Task-92-Sense旧设备能力JSONB兼容迁移", + "path": "docs/task/92-Sense旧设备能力JSONB兼容迁移.md" + }, { "page": "Task-66-Sense视频接入与Profile", "path": "docs/task/66-Sense视频接入与Profile.md"