199 lines
8.3 KiB
Go
199 lines
8.3 KiB
Go
package ops_alert
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"gorm.io/driver/sqlite"
|
|
"gorm.io/gorm"
|
|
|
|
adminModels "git.ilapage.cn/ila/yovision/Sense/server/app/admin/models"
|
|
deviceModels "git.ilapage.cn/ila/yovision/Sense/server/app/sense/device/models"
|
|
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/edge_node"
|
|
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media"
|
|
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media_shard"
|
|
)
|
|
|
|
func opsAlertTestService(t *testing.T) (*Service, *gorm.DB, time.Time) {
|
|
t.Helper()
|
|
db, err := gorm.Open(sqlite.Open("file:"+uuid.NewString()+"?mode=memory&cache=shared"), &gorm.Config{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = db.AutoMigrate(&Alert{}, &Transition{}, &deviceModels.Device{}, &admissionProjection{}, &media.Route{}, &media_shard.Shard{}, &edge_node.Node{}, &adminModels.SysOperaLog{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Date(2026, 8, 28, 8, 0, 0, 0, time.UTC)
|
|
service := NewService(db)
|
|
service.Now = func() time.Time { return now }
|
|
return service, db, now
|
|
}
|
|
|
|
func TestEvaluateSourcesGeneratesSixTypesAndDeduplicates(t *testing.T) {
|
|
service, db, now := opsAlertTestService(t)
|
|
devices := []deviceModels.Device{
|
|
{ID: "offline", Name: "东门摄像机", Location: "东门", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 1},
|
|
{ID: "auth", Name: "仓库摄像机", Location: "仓库", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 2},
|
|
{ID: "clock", Name: "南门摄像机", Location: "南门", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 3},
|
|
}
|
|
if err := db.Create(&devices).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
results := []admissionProjection{
|
|
{DeviceID: "offline", Status: "device_offline", Detail: "设备离线，无法连接", CheckedAt: now},
|
|
{DeviceID: "auth", Status: "authentication_failed", Detail: "设备认证失败", CheckedAt: now},
|
|
{DeviceID: "clock", Status: "clock_skew", Detail: "设备时间漂移", CheckedAt: now},
|
|
}
|
|
if err := db.Create(&results).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Create(&media.Route{ID: "route-1", DeviceID: "auth", ProfileToken: "main", Path: "sense_auth", Desired: "running", Actual: "apply_failed", Detail: "配置未收敛", Version: 4, UpdatedAt: now}).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
probe := now
|
|
if err := db.Create(&media_shard.Shard{ID: "shard-1", Name: "媒体服务 A", Mode: "local", ControlAPI: "http://127.0.0.1:9997", Capacity: 16, Status: media_shard.StatusFailed, Detail: "Control API 不可用", LastProbeAt: &probe, ConfigVersion: 1, CreatedAt: now, UpdatedAt: now}).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Create(&edge_node.Node{ID: "node-1", Name: "主楼边缘节点", Location: "机房", StartedAt: now.Add(-time.Hour), LastHeartbeatAt: now.Add(-10 * time.Second), LastCollectedAt: now.Add(-10 * time.Second), ControlTunnelStatus: edge_node.ChannelUnavailable, ControlTunnelDetail: "TLS 隧道断开", VideoPlaneStatus: edge_node.ChannelReady, ProjectionVersion: 7}).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
response, err := service.EvaluateSources(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.Abnormal != 6 {
|
|
t.Fatalf("abnormal=%d want 6", response.Abnormal)
|
|
}
|
|
var alerts []Alert
|
|
if err = db.Find(&alerts).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(alerts) != 6 {
|
|
t.Fatalf("alerts=%d want 6", len(alerts))
|
|
}
|
|
types := map[string]bool{}
|
|
for _, item := range alerts {
|
|
types[item.AlertType] = true
|
|
if !item.OperationalOnly {
|
|
t.Fatal("alert crossed operational boundary")
|
|
}
|
|
}
|
|
for _, typ := range []string{TypeDeviceOffline, TypeAuthentication, TypeClockDrift, TypeReconciliation, TypeMediaShard, TypeControlTunnel} {
|
|
if !types[typ] {
|
|
t.Fatalf("missing type %s", typ)
|
|
}
|
|
}
|
|
if _, err = service.EvaluateSources(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var count int64
|
|
db.Model(&Alert{}).Count(&count)
|
|
if count != 6 {
|
|
t.Fatalf("duplicate active alerts: %d", count)
|
|
}
|
|
var offline Alert
|
|
if err = db.First(&offline, "fingerprint = ?", TypeDeviceOffline+":device:offline").Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if offline.OccurrenceCount != 1 {
|
|
t.Fatalf("unchanged source counted twice: %d", offline.OccurrenceCount)
|
|
}
|
|
}
|
|
|
|
func TestAcknowledgeRecoverAndReopenPreserveOneAlert(t *testing.T) {
|
|
service, db, now := opsAlertTestService(t)
|
|
device := deviceModels.Device{ID: "auth", Name: "仓库摄像机", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 1}
|
|
if err := db.Create(&device).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result := admissionProjection{DeviceID: "auth", Status: "authentication_failed", Detail: "认证失败", CheckedAt: now}
|
|
if err := db.Create(&result).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := service.EvaluateSources(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var alert Alert
|
|
if err := db.First(&alert, "alert_type = ?", TypeAuthentication).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ack, err := service.Acknowledge(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "已安排现场人员更新设备凭据"}, 7)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ack.State != StateAcknowledged || ack.AcknowledgedBy != 7 {
|
|
t.Fatalf("unexpected acknowledgement: %+v", ack)
|
|
}
|
|
if _, err = service.Acknowledge(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "并发重复确认不应成功"}, 8); !errors.Is(err, ErrVersionConflict) {
|
|
t.Fatalf("want version conflict, got %v", err)
|
|
}
|
|
if err = db.Model(&admissionProjection{}).Where("device_id = ?", "auth").Updates(map[string]any{"status": "ready", "detail": "接入验证完成", "checked_at": now.Add(time.Minute)}).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
service.Now = func() time.Time { return now.Add(time.Minute) }
|
|
if _, err = service.EvaluateSources(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = db.First(&alert, "id = ?", alert.ID).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if alert.State != StateRecovering || alert.HealthySince == nil {
|
|
t.Fatalf("want recovering: %+v", alert)
|
|
}
|
|
if _, err = service.Recover(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "恢复窗口尚未结束，不能关闭"}, 7); !errors.Is(err, ErrRecoveryWindowOpen) {
|
|
t.Fatalf("want recovery window error, got %v", err)
|
|
}
|
|
service.Now = func() time.Time { return now.Add(7 * time.Minute) }
|
|
recovered, err := service.Recover(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "设备已稳定在线超过恢复观察窗口"}, 7)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if recovered.State != StateRecovered {
|
|
t.Fatalf("state=%s", recovered.State)
|
|
}
|
|
if _, err = service.EvaluateSources(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = db.Model(&admissionProjection{}).Where("device_id = ?", "auth").Updates(map[string]any{"status": "authentication_failed", "detail": "认证再次失败", "checked_at": now.Add(8 * time.Minute)}).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
service.Now = func() time.Time { return now.Add(8 * time.Minute) }
|
|
if _, err = service.EvaluateSources(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var count int64
|
|
db.Model(&Alert{}).Where("fingerprint = ?", recovered.Fingerprint).Count(&count)
|
|
if count != 1 {
|
|
t.Fatalf("reopen duplicated alert: %d", count)
|
|
}
|
|
if err = db.First(&alert, "id = ?", alert.ID).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if alert.State != StateUnacknowledged || alert.Cycle != 2 {
|
|
t.Fatalf("unexpected reopened alert: %+v", alert)
|
|
}
|
|
var transitions int64
|
|
db.Model(&Transition{}).Where("alert_id = ?", alert.ID).Count(&transitions)
|
|
if transitions < 5 {
|
|
t.Fatalf("transitions=%d", transitions)
|
|
}
|
|
}
|
|
|
|
func TestAuditIsStoredOnlyInGoAdminOperationLog(t *testing.T) {
|
|
_, db, now := opsAlertTestService(t)
|
|
if err := WriteAudit(db, Audit{Action: "Acknowledge", Method: "POST", Status: "1", Username: "operator", UserID: 9, ClientIP: "127.0.0.1", Route: "/api/v1/ops-alerts/:id/acknowledge", Remark: "确认运维告警", At: now}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var log adminModels.SysOperaLog
|
|
if err := db.First(&log).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if log.Title != "运维告警" || log.CreateBy != 9 {
|
|
t.Fatalf("unexpected audit: %+v", log)
|
|
}
|
|
}
|