feat: 实现 Sense 运维告警隔离 (#79)

Sense/server/app/admin/router/sense_ops_alert.go

@@ -0,0 +1,22 @@
+package router
+
+import (
+	"github.com/gin-gonic/gin"
+	jwt "github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth"
+
+	"git.ilapage.cn/ila/yovision/Sense/server/app/sense/ops_alert"
+	"git.ilapage.cn/ila/yovision/Sense/server/common/actions"
+	"git.ilapage.cn/ila/yovision/Sense/server/common/middleware"
+)
+
+func init() { routerCheckRole = append(routerCheckRole, registerSenseOpsAlertRouter) }
+
+func registerSenseOpsAlertRouter(v1 *gin.RouterGroup, auth *jwt.GinJWTMiddleware) {
+	api := &ops_alert.API{}
+	r := v1.Group("/ops-alerts").Use(auth.MiddlewareFunc()).Use(middleware.AuthCheckRole()).Use(actions.PermissionAction())
+	r.GET("", api.List)
+	r.GET("/:id", api.Get)
+	r.POST("/evaluate", api.Evaluate)
+	r.POST("/:id/acknowledge", api.Acknowledge)
+	r.POST("/:id/recover", api.Recover)
+}

Sense/server/app/admin/router/sense_ops_alert_test.go

@@ -0,0 +1,32 @@
+package router
+
+import (
+	"github.com/gin-gonic/gin"
+	jwt "github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth"
+	"net/http"
+	"testing"
+)
+
+func TestSenseOpsAlertRoutes(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	engine := gin.New()
+	registerSenseOpsAlertRouter(engine.Group("/api/v1"), &jwt.GinJWTMiddleware{})
+	wanted := map[string]bool{
+		http.MethodGet + " /api/v1/ops-alerts":                  false,
+		http.MethodGet + " /api/v1/ops-alerts/:id":              false,
+		http.MethodPost + " /api/v1/ops-alerts/evaluate":        false,
+		http.MethodPost + " /api/v1/ops-alerts/:id/acknowledge": false,
+		http.MethodPost + " /api/v1/ops-alerts/:id/recover":     false,
+	}
+	for _, route := range engine.Routes() {
+		key := route.Method + " " + route.Path
+		if _, ok := wanted[key]; ok {
+			wanted[key] = true
+		}
+	}
+	for route, found := range wanted {
+		if !found {
+			t.Fatalf("route not registered: %s", route)
+		}
+	}
+}

Sense/server/app/sense/ops_alert/apis.go

@@ -0,0 +1,121 @@
+package ops_alert
+
+import (
+	"errors"
+	"git.ilapage.cn/ila/yovision/Sense/server/common"
+	"github.com/gin-gonic/gin"
+	"github.com/gin-gonic/gin/binding"
+	"github.com/go-admin-team/go-admin-core/sdk/api"
+	"github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth/user"
+	coreService "github.com/go-admin-team/go-admin-core/sdk/service"
+	"net/http"
+	"time"
+)
+
+const auditSuccess, auditFailure = "1", "2"
+
+type API struct{ api.Api }
+
+func (e *API) service(c *gin.Context) (*Service, error) {
+	base := coreService.Service{}
+	if err := e.MakeContext(c).MakeOrm().MakeService(&base).Errors; err != nil {
+		return nil, err
+	}
+	return NewService(base.Orm), nil
+}
+func (e *API) List(c *gin.Context) {
+	service, err := e.service(c)
+	if err != nil {
+		e.writeError(err)
+		return
+	}
+	request := PageRequest{}
+	if err = e.MakeContext(c).Bind(&request).Errors; err != nil {
+		e.audit(c, service, "List", auditFailure, "告警查询条件格式不正确")
+		e.Error(http.StatusBadRequest, err, "查询条件格式不正确")
+		return
+	}
+	response, err := service.List(request)
+	if err != nil {
+		e.audit(c, service, "List", auditFailure, "告警列表查询失败")
+		e.writeError(err)
+		return
+	}
+	e.audit(c, service, "List", auditSuccess, "读取运维告警列表")
+	e.OK(response, "查询成功")
+}
+func (e *API) Get(c *gin.Context) {
+	service, err := e.service(c)
+	if err != nil {
+		e.writeError(err)
+		return
+	}
+	response, err := service.Get(c.Param("id"))
+	if err != nil {
+		e.audit(c, service, "Get", auditFailure, "告警详情查询失败")
+		e.writeError(err)
+		return
+	}
+	e.audit(c, service, "Get", auditSuccess, "读取运维告警详情 "+response.Alert.ID)
+	e.OK(response, "查询成功")
+}
+func (e *API) Evaluate(c *gin.Context) {
+	service, err := e.service(c)
+	if err != nil {
+		e.writeError(err)
+		return
+	}
+	response, err := service.EvaluateSources(c.Request.Context())
+	if err != nil {
+		e.audit(c, service, "Evaluate", auditFailure, "健康事实刷新失败")
+		e.writeError(err)
+		return
+	}
+	e.audit(c, service, "Evaluate", auditSuccess, "刷新 Sense 内部健康事实")
+	e.OK(response, "状态已刷新")
+}
+func (e *API) Acknowledge(c *gin.Context) { e.action(c, "Acknowledge") }
+func (e *API) Recover(c *gin.Context)     { e.action(c, "Recover") }
+func (e *API) action(c *gin.Context, action string) {
+	service, err := e.service(c)
+	if err != nil {
+		e.writeError(err)
+		return
+	}
+	request := ActionRequest{}
+	if err = e.MakeContext(c).Bind(&request, binding.JSON).Errors; err != nil {
+		e.audit(c, service, action, auditFailure, "告警操作请求格式不正确")
+		e.Error(http.StatusBadRequest, err, "请求格式不正确")
+		return
+	}
+	var item Alert
+	if action == "Acknowledge" {
+		item, err = service.Acknowledge(c.Request.Context(), c.Param("id"), request, user.GetUserId(c))
+	} else {
+		item, err = service.Recover(c.Request.Context(), c.Param("id"), request, user.GetUserId(c))
+	}
+	if err != nil {
+		e.audit(c, service, action, auditFailure, "运维告警状态操作被拒绝")
+		e.writeError(err)
+		return
+	}
+	e.audit(c, service, action, auditSuccess, action+" "+item.ID)
+	e.OK(item, "操作成功")
+}
+func (e *API) audit(c *gin.Context, service *Service, action, status, remark string) {
+	if err := WriteAudit(service.DB, Audit{Action: action, Method: c.Request.Method, Status: status, Username: user.GetUserName(c), UserID: user.GetUserId(c), ClientIP: common.GetClientIP(c), Route: c.FullPath(), Remark: remark, At: time.Now()}); err != nil {
+		api.GetRequestLogger(c).Errorf("ops alert audit failed: %s", err.Error())
+	}
+}
+func (e *API) writeError(err error) {
+	switch {
+	case errors.Is(err, ErrInvalidFilter), errors.Is(err, ErrInvalidReason):
+		e.Error(http.StatusBadRequest, err, err.Error())
+	case errors.Is(err, ErrVersionConflict), errors.Is(err, ErrInvalidTransition), errors.Is(err, ErrRecoveryWindowOpen):
+		e.Error(http.StatusConflict, err, err.Error())
+	case errors.Is(err, ErrAlertNotFound):
+		e.Error(http.StatusNotFound, err, err.Error())
+	default:
+		e.Error(http.StatusInternalServerError, err, "运维告警操作失败")
+	}
+}

Sense/server/app/sense/ops_alert/audit.go

@@ -0,0 +1,19 @@
+package ops_alert
+
+import (
+	adminModels "git.ilapage.cn/ila/yovision/Sense/server/app/admin/models"
+	"gorm.io/gorm"
+	"time"
+)
+
+type Audit struct {
+	Action, Method, Status, Username, ClientIP, Route, Remark string
+	UserID                                                    int
+	At                                                        time.Time
+}
+
+func WriteAudit(db *gorm.DB, input Audit) error {
+	model := adminModels.SysOperaLog{Title: "运维告警", BusinessType: "other", Method: "ops_alert.API." + input.Action, RequestMethod: input.Method, OperatorType: "1", OperName: input.Username, OperUrl: input.Route, OperIp: input.ClientIP, Status: input.Status, OperTime: input.At.UTC(), Remark: input.Remark, CreatedAt: input.At.UTC(), UpdatedAt: input.At.UTC()}
+	model.CreateBy, model.UpdateBy = input.UserID, input.UserID
+	return db.Create(&model).Error
+}

Sense/server/app/sense/ops_alert/dto.go

@@ -0,0 +1,37 @@
+package ops_alert
+
+import commonDto "git.ilapage.cn/ila/yovision/Sense/server/common/dto"
+
+type PageRequest struct {
+	commonDto.Pagination `search:"-"`
+	AlertType            string `form:"alertType"`
+	State                string `form:"state"`
+	Keyword              string `form:"keyword"`
+}
+
+type Summary struct {
+	Active         int64 `json:"active"`
+	Unacknowledged int64 `json:"unacknowledged"`
+	Recovering     int64 `json:"recovering"`
+}
+
+type PageResponse struct {
+	Summary Summary `json:"summary"`
+	List    []Alert `json:"list"`
+	Count   int64   `json:"count"`
+}
+
+type DetailResponse struct {
+	Alert       Alert        `json:"alert"`
+	Transitions []Transition `json:"transitions"`
+}
+
+type ActionRequest struct {
+	ExpectedVersion int64  `json:"expectedVersion" binding:"required,min=1"`
+	Reason          string `json:"reason" binding:"required"`
+}
+
+type EvaluateResponse struct {
+	Observed int `json:"observed"`
+	Abnormal int `json:"abnormal"`
+}

Sense/server/app/sense/ops_alert/models.go

@@ -0,0 +1,62 @@
+package ops_alert
+
+import "time"
+
+const (
+	StateUnacknowledged = "unacknowledged"
+	StateAcknowledged   = "acknowledged"
+	StateRecovering     = "recovering"
+	StateRecovered      = "recovered"
+
+	TypeDeviceOffline  = "device_offline"
+	TypeAuthentication = "authentication_failed"
+	TypeClockDrift     = "clock_drift"
+	TypeReconciliation = "reconciliation_failed"
+	TypeMediaShard     = "media_shard_failed"
+	TypeControlTunnel  = "control_tunnel_failed"
+)
+
+type Alert struct {
+	ID                string     `gorm:"size:36;primaryKey" json:"id"`
+	Fingerprint       string     `gorm:"size:255;not null;uniqueIndex" json:"fingerprint"`
+	AlertType         string     `gorm:"size:32;not null;index" json:"alertType"`
+	Severity          string     `gorm:"size:16;not null;index" json:"severity"`
+	ObjectType        string     `gorm:"size:32;not null;index" json:"objectType"`
+	ObjectID          string     `gorm:"size:96;not null;index" json:"objectId"`
+	ObjectName        string     `gorm:"size:128;not null" json:"objectName"`
+	Location          string     `gorm:"size:255;not null;default:''" json:"location"`
+	State             string     `gorm:"size:24;not null;index" json:"state"`
+	Title             string     `gorm:"size:160;not null" json:"title"`
+	Detail            string     `gorm:"size:512;not null" json:"detail"`
+	NextAction        string     `gorm:"size:255;not null" json:"nextAction"`
+	LastSourceVersion string     `gorm:"size:128;not null;default:''" json:"-"`
+	OccurrenceCount   int        `gorm:"not null;default:1" json:"occurrenceCount"`
+	Cycle             int        `gorm:"not null;default:1" json:"cycle"`
+	FirstSeenAt       time.Time  `gorm:"not null;index" json:"firstSeenAt"`
+	LastSeenAt        time.Time  `gorm:"not null;index" json:"lastSeenAt"`
+	HealthySince      *time.Time `json:"healthySince,omitempty"`
+	AcknowledgedAt    *time.Time `json:"acknowledgedAt,omitempty"`
+	AcknowledgedBy    int        `gorm:"not null;default:0" json:"acknowledgedBy,omitempty"`
+	RecoveredAt       *time.Time `json:"recoveredAt,omitempty"`
+	RecoveredBy       int        `gorm:"not null;default:0" json:"recoveredBy,omitempty"`
+	OperationalOnly   bool       `gorm:"not null;default:true" json:"operationalOnly"`
+	Version           int64      `gorm:"not null;default:1" json:"version"`
+	CreatedAt         time.Time  `gorm:"not null" json:"createdAt"`
+	UpdatedAt         time.Time  `gorm:"not null" json:"updatedAt"`
+}
+
+func (Alert) TableName() string { return "sense_ops_alerts" }
+
+type Transition struct {
+	ID          string    `gorm:"size:36;primaryKey" json:"id"`
+	AlertID     string    `gorm:"size:36;not null;index" json:"alertId"`
+	Cycle       int       `gorm:"not null" json:"cycle"`
+	Action      string    `gorm:"size:32;not null;index" json:"action"`
+	FromState   string    `gorm:"size:24;not null;default:''" json:"fromState"`
+	ToState     string    `gorm:"size:24;not null" json:"toState"`
+	Reason      string    `gorm:"size:512;not null;default:''" json:"reason"`
+	ActorUserID int       `gorm:"not null;default:0" json:"actorUserId"`
+	CreatedAt   time.Time `gorm:"not null;index" json:"createdAt"`
+}
+
+func (Transition) TableName() string { return "sense_ops_alert_transitions" }

Sense/server/app/sense/ops_alert/service.go

@@ -0,0 +1,326 @@
+package ops_alert
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/google/uuid"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+
+	deviceModels "git.ilapage.cn/ila/yovision/Sense/server/app/sense/device/models"
+	"git.ilapage.cn/ila/yovision/Sense/server/app/sense/edge_node"
+	"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media"
+	"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media_shard"
+)
+
+var (
+	ErrAlertNotFound      = errors.New("运维告警不存在")
+	ErrInvalidFilter      = errors.New("告警筛选条件不正确")
+	ErrInvalidReason      = errors.New("处理说明需为 6 至 256 个字符")
+	ErrInvalidTransition  = errors.New("当前告警状态不允许此操作")
+	ErrVersionConflict    = errors.New("告警状态已变化，请刷新后重试")
+	ErrRecoveryWindowOpen = errors.New("健康恢复观察窗口尚未结束")
+)
+
+const defaultRecoveryWindow = 5 * time.Minute
+
+type Service struct {
+	DB             *gorm.DB
+	Now            func() time.Time
+	RecoveryWindow time.Duration
+}
+
+func NewService(db *gorm.DB) *Service {
+	return &Service{DB: db, Now: time.Now, RecoveryWindow: defaultRecoveryWindow}
+}
+
+func (s *Service) List(request PageRequest) (PageResponse, error) {
+	if !validOptional(request.AlertType, alertTypes()) || !validOptional(request.State, states()) {
+		return PageResponse{}, ErrInvalidFilter
+	}
+	query := s.DB.Model(&Alert{})
+	if request.AlertType != "" {
+		query = query.Where("alert_type = ?", request.AlertType)
+	}
+	if request.State != "" {
+		query = query.Where("state = ?", request.State)
+	}
+	if keyword := strings.TrimSpace(request.Keyword); keyword != "" {
+		like := "%" + keyword + "%"
+		query = query.Where("object_name LIKE ? OR location LIKE ? OR title LIKE ? OR detail LIKE ?", like, like, like, like)
+	}
+	response := PageResponse{}
+	if err := query.Count(&response.Count).Error; err != nil {
+		return response, err
+	}
+	if err := query.Order("CASE WHEN state = 'unacknowledged' THEN 0 WHEN state = 'acknowledged' THEN 1 WHEN state = 'recovering' THEN 2 ELSE 3 END, last_seen_at DESC").Offset((request.GetPageIndex() - 1) * request.GetPageSize()).Limit(request.GetPageSize()).Find(&response.List).Error; err != nil {
+		return response, err
+	}
+	active := []string{StateUnacknowledged, StateAcknowledged, StateRecovering}
+	if err := s.DB.Model(&Alert{}).Where("state IN ?", active).Count(&response.Summary.Active).Error; err != nil {
+		return response, err
+	}
+	if err := s.DB.Model(&Alert{}).Where("state = ?", StateUnacknowledged).Count(&response.Summary.Unacknowledged).Error; err != nil {
+		return response, err
+	}
+	if err := s.DB.Model(&Alert{}).Where("state = ?", StateRecovering).Count(&response.Summary.Recovering).Error; err != nil {
+		return response, err
+	}
+	return response, nil
+}
+
+func (s *Service) Get(id string) (DetailResponse, error) {
+	response := DetailResponse{Transitions: []Transition{}}
+	if err := s.DB.First(&response.Alert, "id = ?", id).Error; errors.Is(err, gorm.ErrRecordNotFound) {
+		return response, ErrAlertNotFound
+	} else if err != nil {
+		return response, err
+	}
+	if err := s.DB.Where("alert_id = ?", id).Order("created_at DESC").Find(&response.Transitions).Error; err != nil {
+		return response, err
+	}
+	return response, nil
+}
+
+func (s *Service) Acknowledge(ctx context.Context, id string, request ActionRequest, actor int) (Alert, error) {
+	return s.transition(ctx, id, request, actor, StateUnacknowledged, StateAcknowledged, "acknowledge")
+}
+
+func (s *Service) Recover(ctx context.Context, id string, request ActionRequest, actor int) (Alert, error) {
+	return s.transition(ctx, id, request, actor, StateRecovering, StateRecovered, "recover")
+}
+
+func (s *Service) transition(ctx context.Context, id string, request ActionRequest, actor int, expectedState, targetState, action string) (Alert, error) {
+	reason := strings.TrimSpace(request.Reason)
+	if utf8.RuneCountInString(reason) < 6 || utf8.RuneCountInString(reason) > 256 {
+		return Alert{}, ErrInvalidReason
+	}
+	if request.ExpectedVersion < 1 {
+		return Alert{}, ErrVersionConflict
+	}
+	now := s.Now().UTC()
+	result := Alert{}
+	err := s.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+		var current Alert
+		if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&current, "id = ?", id).Error; errors.Is(err, gorm.ErrRecordNotFound) {
+			return ErrAlertNotFound
+		} else if err != nil {
+			return err
+		}
+		if current.Version != request.ExpectedVersion {
+			return ErrVersionConflict
+		}
+		if current.State != expectedState {
+			return ErrInvalidTransition
+		}
+		if action == "recover" && (current.HealthySince == nil || now.Sub(current.HealthySince.UTC()) < s.RecoveryWindow) {
+			return ErrRecoveryWindowOpen
+		}
+		updates := map[string]any{"state": targetState, "version": current.Version + 1, "updated_at": now}
+		if action == "acknowledge" {
+			updates["acknowledged_at"], updates["acknowledged_by"] = now, actor
+		} else {
+			updates["recovered_at"], updates["recovered_by"] = now, actor
+		}
+		write := tx.Model(&Alert{}).Where("id = ? AND version = ? AND state = ?", id, current.Version, expectedState).Updates(updates)
+		if write.Error != nil {
+			return write.Error
+		}
+		if write.RowsAffected != 1 {
+			return ErrVersionConflict
+		}
+		if err := tx.Create(&Transition{ID: uuid.NewString(), AlertID: id, Cycle: current.Cycle, Action: action, FromState: expectedState, ToState: targetState, Reason: reason, ActorUserID: actor, CreatedAt: now}).Error; err != nil {
+			return err
+		}
+		return tx.First(&result, "id = ?", id).Error
+	})
+	return result, err
+}
+
+type observation struct {
+	Fingerprint, AlertType, Severity, ObjectType, ObjectID, ObjectName, Location, Title, Detail, NextAction, SourceVersion string
+	Abnormal                                                                                                               bool
+}
+
+func (s *Service) EvaluateSources(ctx context.Context) (EvaluateResponse, error) {
+	items, err := s.sourceObservations(ctx)
+	if err != nil {
+		return EvaluateResponse{}, err
+	}
+	response := EvaluateResponse{Observed: len(items)}
+	for _, item := range items {
+		if item.Abnormal {
+			response.Abnormal++
+		}
+		if err = s.observe(ctx, item); err != nil {
+			return response, err
+		}
+	}
+	return response, nil
+}
+
+func (s *Service) observe(ctx context.Context, item observation) error {
+	now := s.Now().UTC()
+	return s.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+		var current Alert
+		err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("fingerprint = ?", item.Fingerprint).First(&current).Error
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			if !item.Abnormal {
+				return nil
+			}
+			current = Alert{ID: uuid.NewString(), Fingerprint: item.Fingerprint, AlertType: item.AlertType, Severity: item.Severity, ObjectType: item.ObjectType, ObjectID: item.ObjectID, ObjectName: item.ObjectName, Location: item.Location, State: StateUnacknowledged, Title: item.Title, Detail: item.Detail, NextAction: item.NextAction, LastSourceVersion: item.SourceVersion, OccurrenceCount: 1, Cycle: 1, FirstSeenAt: now, LastSeenAt: now, OperationalOnly: true, Version: 1, CreatedAt: now, UpdatedAt: now}
+			if err = tx.Create(&current).Error; err != nil {
+				return err
+			}
+			return tx.Create(&Transition{ID: uuid.NewString(), AlertID: current.ID, Cycle: 1, Action: "detected", ToState: StateUnacknowledged, Reason: "健康事实首次满足告警规则", CreatedAt: now}).Error
+		}
+		if err != nil {
+			return err
+		}
+		if item.Abnormal {
+			updates := map[string]any{"severity": item.Severity, "object_name": item.ObjectName, "location": item.Location, "title": item.Title, "detail": item.Detail, "next_action": item.NextAction, "last_seen_at": now, "updated_at": now, "healthy_since": nil}
+			if item.SourceVersion != current.LastSourceVersion {
+				updates["last_source_version"], updates["occurrence_count"] = item.SourceVersion, current.OccurrenceCount+1
+			}
+			if current.State == StateRecovered {
+				updates["state"], updates["cycle"], updates["first_seen_at"], updates["occurrence_count"], updates["acknowledged_at"], updates["acknowledged_by"], updates["recovered_at"], updates["recovered_by"] = StateUnacknowledged, current.Cycle+1, now, 1, nil, 0, nil, 0
+				if err = tx.Create(&Transition{ID: uuid.NewString(), AlertID: current.ID, Cycle: current.Cycle + 1, Action: "reopened", FromState: StateRecovered, ToState: StateUnacknowledged, Reason: "健康事实再次异常", CreatedAt: now}).Error; err != nil {
+					return err
+				}
+			} else if current.State == StateRecovering {
+				target := StateUnacknowledged
+				if current.AcknowledgedAt != nil {
+					target = StateAcknowledged
+				}
+				updates["state"] = target
+				if err = tx.Create(&Transition{ID: uuid.NewString(), AlertID: current.ID, Cycle: current.Cycle, Action: "relapsed", FromState: StateRecovering, ToState: target, Reason: "恢复观察期间再次异常", CreatedAt: now}).Error; err != nil {
+					return err
+				}
+			}
+			updates["version"] = current.Version + 1
+			return tx.Model(&Alert{}).Where("id = ? AND version = ?", current.ID, current.Version).Updates(updates).Error
+		}
+		if current.State == StateUnacknowledged || current.State == StateAcknowledged {
+			if err = tx.Create(&Transition{ID: uuid.NewString(), AlertID: current.ID, Cycle: current.Cycle, Action: "health_restored", FromState: current.State, ToState: StateRecovering, Reason: "健康事实已恢复，进入稳定观察窗口", CreatedAt: now}).Error; err != nil {
+				return err
+			}
+			return tx.Model(&Alert{}).Where("id = ? AND version = ?", current.ID, current.Version).Updates(map[string]any{"state": StateRecovering, "healthy_since": now, "version": current.Version + 1, "updated_at": now}).Error
+		}
+		return nil
+	})
+}
+
+type admissionProjection struct {
+	DeviceID, Status, Detail string
+	CheckedAt                time.Time
+}
+
+func (admissionProjection) TableName() string { return "sense_admission_results" }
+
+func (s *Service) sourceObservations(ctx context.Context) ([]observation, error) {
+	now := s.Now().UTC()
+	result := []observation{}
+	var devices []deviceModels.Device
+	if err := s.DB.WithContext(ctx).Find(&devices).Error; err != nil {
+		return nil, err
+	}
+	var admissions []admissionProjection
+	if err := s.DB.WithContext(ctx).Find(&admissions).Error; err != nil && !isMissingTable(err) {
+		return nil, err
+	}
+	byDevice := map[string]admissionProjection{}
+	for _, item := range admissions {
+		byDevice[item.DeviceID] = item
+	}
+	for _, device := range devices {
+		if device.Status == deviceModels.StatusDisabled {
+			continue
+		}
+		state, detail, observed := device.AdapterStatus, "尚未获得设备接入结果", device.UpdatedAt
+		if item, ok := byDevice[device.ID]; ok {
+			state, detail, observed = item.Status, item.Detail, item.CheckedAt
+		}
+		text := strings.ToLower(state + " " + detail)
+		version := fmt.Sprintf("%d:%d", device.Version, observed.UnixNano())
+		base := observation{ObjectType: "device", ObjectID: device.ID, ObjectName: device.Name, Location: device.Location, SourceVersion: version}
+		result = append(result,
+			withRule(base, TypeDeviceOffline, "high", "设备离线", detail, "检查设备供电、网络和地址后重新发现", containsAny(text, "offline", "unreachable", "timeout", "离线", "无法连接", "超时")),
+			withRule(base, TypeAuthentication, "high", "设备认证失败", detail, "更新设备凭据后重新执行接入验证", containsAny(text, "authentication_failed", "unauthorized", "auth", "认证", "凭据")),
+			withRule(base, TypeClockDrift, "medium", "设备时间漂移", detail, "校准设备时间后重新执行接入验证", containsAny(text, "clock_skew", "clock_drift", "time drift", "时间漂移", "时钟")),
+		)
+	}
+	var routes []media.Route
+	if err := s.DB.WithContext(ctx).Find(&routes).Error; err != nil && !isMissingTable(err) {
+		return nil, err
+	}
+	for _, route := range routes {
+		converged := (route.Desired == media.DesiredRunning && (route.Actual == "ready" || route.Actual == "waiting")) || (route.Desired == media.DesiredStopped && route.Actual == "stopped")
+		base := observation{ObjectType: "media_route", ObjectID: route.ID, ObjectName: route.Path, SourceVersion: fmt.Sprintf("%d", route.Version)}
+		result = append(result, withRule(base, TypeReconciliation, "medium", "媒体状态未收敛", route.Detail, "在运维中心核对期望态并执行受控重试", !converged))
+	}
+	var shards []media_shard.Shard
+	if err := s.DB.WithContext(ctx).Find(&shards).Error; err != nil && !isMissingTable(err) {
+		return nil, err
+	}
+	for _, shard := range shards {
+		if shard.Status == media_shard.StatusDisabled {
+			continue
+		}
+		stale := shard.LastProbeAt == nil || now.Sub(shard.LastProbeAt.UTC()) > 30*time.Second
+		base := observation{ObjectType: "media_shard", ObjectID: shard.ID, ObjectName: shard.Name, SourceVersion: fmt.Sprintf("%d:%d", shard.ConfigVersion, timeValue(shard.LastProbeAt))}
+		result = append(result, withRule(base, TypeMediaShard, "high", "媒体分片异常", shard.Detail, "检查 MediaMTX 进程和 Control API 后重新探测", shard.Status == media_shard.StatusFailed || stale))
+	}
+	var nodes []edge_node.Node
+	if err := s.DB.WithContext(ctx).Find(&nodes).Error; err != nil && !isMissingTable(err) {
+		return nil, err
+	}
+	for _, node := range nodes {
+		base := observation{ObjectType: "edge_node", ObjectID: node.ID, ObjectName: node.Name, Location: node.Location, SourceVersion: fmt.Sprintf("%d:%d", node.ProjectionVersion, node.LastHeartbeatAt.UnixNano())}
+		result = append(result,
+			withRule(base, TypeDeviceOffline, "high", "边缘节点离线", "心跳超过 90 秒未更新", "检查节点进程、网络和机器身份后等待心跳恢复", now.Sub(node.LastHeartbeatAt.UTC()) > edge_node.HeartbeatTimeout),
+			withRule(base, TypeControlTunnel, "high", "控制隧道异常", node.ControlTunnelDetail, "检查节点到 Sense 的控制通道，不影响业务预警数据", node.ControlTunnelStatus != edge_node.ChannelReady),
+		)
+	}
+	return result, nil
+}
+
+func withRule(base observation, typ, severity, title, detail, next string, abnormal bool) observation {
+	base.Fingerprint = typ + ":" + base.ObjectType + ":" + base.ObjectID
+	base.AlertType = typ
+	base.Severity = severity
+	base.Title = title
+	base.Detail = detail
+	base.NextAction = next
+	base.Abnormal = abnormal
+	return base
+}
+func containsAny(value string, needles ...string) bool {
+	for _, needle := range needles {
+		if strings.Contains(value, needle) {
+			return true
+		}
+	}
+	return false
+}
+func isMissingTable(err error) bool {
+	text := strings.ToLower(err.Error())
+	return strings.Contains(text, "no such table") || strings.Contains(text, "does not exist")
+}
+func timeValue(value *time.Time) int64 {
+	if value == nil {
+		return 0
+	}
+	return value.UnixNano()
+}
+func validOptional(value string, allowed map[string]bool) bool { return value == "" || allowed[value] }
+func states() map[string]bool {
+	return map[string]bool{StateUnacknowledged: true, StateAcknowledged: true, StateRecovering: true, StateRecovered: true}
+}
+func alertTypes() map[string]bool {
+	return map[string]bool{TypeDeviceOffline: true, TypeAuthentication: true, TypeClockDrift: true, TypeReconciliation: true, TypeMediaShard: true, TypeControlTunnel: true}
+}

Sense/server/app/sense/ops_alert/service_test.go

@@ -0,0 +1,198 @@
+package ops_alert
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+
+	adminModels "git.ilapage.cn/ila/yovision/Sense/server/app/admin/models"
+	deviceModels "git.ilapage.cn/ila/yovision/Sense/server/app/sense/device/models"
+	"git.ilapage.cn/ila/yovision/Sense/server/app/sense/edge_node"
+	"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media"
+	"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media_shard"
+)
+
+func opsAlertTestService(t *testing.T) (*Service, *gorm.DB, time.Time) {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open("file:"+uuid.NewString()+"?mode=memory&cache=shared"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = db.AutoMigrate(&Alert{}, &Transition{}, &deviceModels.Device{}, &admissionProjection{}, &media.Route{}, &media_shard.Shard{}, &edge_node.Node{}, &adminModels.SysOperaLog{}); err != nil {
+		t.Fatal(err)
+	}
+	now := time.Date(2026, 8, 28, 8, 0, 0, 0, time.UTC)
+	service := NewService(db)
+	service.Now = func() time.Time { return now }
+	return service, db, now
+}
+
+func TestEvaluateSourcesGeneratesSixTypesAndDeduplicates(t *testing.T) {
+	service, db, now := opsAlertTestService(t)
+	devices := []deviceModels.Device{
+		{ID: "offline", Name: "东门摄像机", Location: "东门", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 1},
+		{ID: "auth", Name: "仓库摄像机", Location: "仓库", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 2},
+		{ID: "clock", Name: "南门摄像机", Location: "南门", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 3},
+	}
+	if err := db.Create(&devices).Error; err != nil {
+		t.Fatal(err)
+	}
+	results := []admissionProjection{
+		{DeviceID: "offline", Status: "device_offline", Detail: "设备离线，无法连接", CheckedAt: now},
+		{DeviceID: "auth", Status: "authentication_failed", Detail: "设备认证失败", CheckedAt: now},
+		{DeviceID: "clock", Status: "clock_skew", Detail: "设备时间漂移", CheckedAt: now},
+	}
+	if err := db.Create(&results).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Create(&media.Route{ID: "route-1", DeviceID: "auth", ProfileToken: "main", Path: "sense_auth", Desired: "running", Actual: "apply_failed", Detail: "配置未收敛", Version: 4, UpdatedAt: now}).Error; err != nil {
+		t.Fatal(err)
+	}
+	probe := now
+	if err := db.Create(&media_shard.Shard{ID: "shard-1", Name: "媒体服务 A", Mode: "local", ControlAPI: "http://127.0.0.1:9997", Capacity: 16, Status: media_shard.StatusFailed, Detail: "Control API 不可用", LastProbeAt: &probe, ConfigVersion: 1, CreatedAt: now, UpdatedAt: now}).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Create(&edge_node.Node{ID: "node-1", Name: "主楼边缘节点", Location: "机房", StartedAt: now.Add(-time.Hour), LastHeartbeatAt: now.Add(-10 * time.Second), LastCollectedAt: now.Add(-10 * time.Second), ControlTunnelStatus: edge_node.ChannelUnavailable, ControlTunnelDetail: "TLS 隧道断开", VideoPlaneStatus: edge_node.ChannelReady, ProjectionVersion: 7}).Error; err != nil {
+		t.Fatal(err)
+	}
+
+	response, err := service.EvaluateSources(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response.Abnormal != 6 {
+		t.Fatalf("abnormal=%d want 6", response.Abnormal)
+	}
+	var alerts []Alert
+	if err = db.Find(&alerts).Error; err != nil {
+		t.Fatal(err)
+	}
+	if len(alerts) != 6 {
+		t.Fatalf("alerts=%d want 6", len(alerts))
+	}
+	types := map[string]bool{}
+	for _, item := range alerts {
+		types[item.AlertType] = true
+		if !item.OperationalOnly {
+			t.Fatal("alert crossed operational boundary")
+		}
+	}
+	for _, typ := range []string{TypeDeviceOffline, TypeAuthentication, TypeClockDrift, TypeReconciliation, TypeMediaShard, TypeControlTunnel} {
+		if !types[typ] {
+			t.Fatalf("missing type %s", typ)
+		}
+	}
+	if _, err = service.EvaluateSources(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	var count int64
+	db.Model(&Alert{}).Count(&count)
+	if count != 6 {
+		t.Fatalf("duplicate active alerts: %d", count)
+	}
+	var offline Alert
+	if err = db.First(&offline, "fingerprint = ?", TypeDeviceOffline+":device:offline").Error; err != nil {
+		t.Fatal(err)
+	}
+	if offline.OccurrenceCount != 1 {
+		t.Fatalf("unchanged source counted twice: %d", offline.OccurrenceCount)
+	}
+}
+
+func TestAcknowledgeRecoverAndReopenPreserveOneAlert(t *testing.T) {
+	service, db, now := opsAlertTestService(t)
+	device := deviceModels.Device{ID: "auth", Name: "仓库摄像机", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 1}
+	if err := db.Create(&device).Error; err != nil {
+		t.Fatal(err)
+	}
+	result := admissionProjection{DeviceID: "auth", Status: "authentication_failed", Detail: "认证失败", CheckedAt: now}
+	if err := db.Create(&result).Error; err != nil {
+		t.Fatal(err)
+	}
+	if _, err := service.EvaluateSources(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	var alert Alert
+	if err := db.First(&alert, "alert_type = ?", TypeAuthentication).Error; err != nil {
+		t.Fatal(err)
+	}
+	ack, err := service.Acknowledge(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "已安排现场人员更新设备凭据"}, 7)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ack.State != StateAcknowledged || ack.AcknowledgedBy != 7 {
+		t.Fatalf("unexpected acknowledgement: %+v", ack)
+	}
+	if _, err = service.Acknowledge(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "并发重复确认不应成功"}, 8); !errors.Is(err, ErrVersionConflict) {
+		t.Fatalf("want version conflict, got %v", err)
+	}
+	if err = db.Model(&admissionProjection{}).Where("device_id = ?", "auth").Updates(map[string]any{"status": "ready", "detail": "接入验证完成", "checked_at": now.Add(time.Minute)}).Error; err != nil {
+		t.Fatal(err)
+	}
+	service.Now = func() time.Time { return now.Add(time.Minute) }
+	if _, err = service.EvaluateSources(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if err = db.First(&alert, "id = ?", alert.ID).Error; err != nil {
+		t.Fatal(err)
+	}
+	if alert.State != StateRecovering || alert.HealthySince == nil {
+		t.Fatalf("want recovering: %+v", alert)
+	}
+	if _, err = service.Recover(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "恢复窗口尚未结束，不能关闭"}, 7); !errors.Is(err, ErrRecoveryWindowOpen) {
+		t.Fatalf("want recovery window error, got %v", err)
+	}
+	service.Now = func() time.Time { return now.Add(7 * time.Minute) }
+	recovered, err := service.Recover(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "设备已稳定在线超过恢复观察窗口"}, 7)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if recovered.State != StateRecovered {
+		t.Fatalf("state=%s", recovered.State)
+	}
+	if _, err = service.EvaluateSources(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if err = db.Model(&admissionProjection{}).Where("device_id = ?", "auth").Updates(map[string]any{"status": "authentication_failed", "detail": "认证再次失败", "checked_at": now.Add(8 * time.Minute)}).Error; err != nil {
+		t.Fatal(err)
+	}
+	service.Now = func() time.Time { return now.Add(8 * time.Minute) }
+	if _, err = service.EvaluateSources(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	var count int64
+	db.Model(&Alert{}).Where("fingerprint = ?", recovered.Fingerprint).Count(&count)
+	if count != 1 {
+		t.Fatalf("reopen duplicated alert: %d", count)
+	}
+	if err = db.First(&alert, "id = ?", alert.ID).Error; err != nil {
+		t.Fatal(err)
+	}
+	if alert.State != StateUnacknowledged || alert.Cycle != 2 {
+		t.Fatalf("unexpected reopened alert: %+v", alert)
+	}
+	var transitions int64
+	db.Model(&Transition{}).Where("alert_id = ?", alert.ID).Count(&transitions)
+	if transitions < 5 {
+		t.Fatalf("transitions=%d", transitions)
+	}
+}
+
+func TestAuditIsStoredOnlyInGoAdminOperationLog(t *testing.T) {
+	_, db, now := opsAlertTestService(t)
+	if err := WriteAudit(db, Audit{Action: "Acknowledge", Method: "POST", Status: "1", Username: "operator", UserID: 9, ClientIP: "127.0.0.1", Route: "/api/v1/ops-alerts/:id/acknowledge", Remark: "确认运维告警", At: now}); err != nil {
+		t.Fatal(err)
+	}
+	var log adminModels.SysOperaLog
+	if err := db.First(&log).Error; err != nil {
+		t.Fatal(err)
+	}
+	if log.Title != "运维告警" || log.CreateBy != 9 {
+		t.Fatalf("unexpected audit: %+v", log)
+	}
+}

Sense/server/cmd/migrate/migration/version/2026082816000_ops_alert.go

@@ -0,0 +1,74 @@
+package version
+
+import (
+	"fmt"
+	"git.ilapage.cn/ila/yovision/Sense/server/app/sense/ops_alert"
+	"git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration"
+	migrationModels "git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration/models"
+	common "git.ilapage.cn/ila/yovision/Sense/server/common/models"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+	"runtime"
+)
+
+func init() {
+	_, fileName, _, _ := runtime.Caller(0)
+	migration.Migrate.SetVersion(migration.GetFilename(fileName), migrateSenseOpsAlert)
+}
+
+func migrateSenseOpsAlert(db *gorm.DB, version string) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+		if err := tx.AutoMigrate(&ops_alert.Alert{}, &ops_alert.Transition{}); err != nil {
+			return err
+		}
+		root, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: senseLayoutMenuName, Title: "视频感知", Icon: "video-camera", Path: "/sense", MenuType: "M", Component: "Layout", Sort: 5, Visible: "0", IsFrame: "1"})
+		if err != nil {
+			return err
+		}
+		page, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlert", Title: "运维告警", Icon: "warning", Path: "ops-alert", Paths: fmt.Sprintf("/0/%d", root.MenuId), MenuType: "C", Permission: "sense:ops-alert:list", ParentId: root.MenuId, Component: "/sense/ops-alert/index", Sort: 11, Visible: "0", IsFrame: "1"})
+		if err != nil {
+			return err
+		}
+		detail, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlertDetail", Title: "查看告警详情", MenuType: "F", Action: "GET", Permission: "sense:ops-alert:detail", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 1, Visible: "1", IsFrame: "1"})
+		if err != nil {
+			return err
+		}
+		refresh, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlertEvaluate", Title: "刷新告警状态", MenuType: "F", Action: "POST", Permission: "sense:ops-alert:evaluate", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 2, Visible: "1", IsFrame: "1"})
+		if err != nil {
+			return err
+		}
+		ack, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlertAcknowledge", Title: "确认运维告警", MenuType: "F", Action: "POST", Permission: "sense:ops-alert:acknowledge", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 3, Visible: "1", IsFrame: "1"})
+		if err != nil {
+			return err
+		}
+		recover, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlertRecover", Title: "确认告警恢复", MenuType: "F", Action: "POST", Permission: "sense:ops-alert:recover", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 4, Visible: "1", IsFrame: "1"})
+		if err != nil {
+			return err
+		}
+		readPolicies := [][2]string{{"/api/v1/ops-alerts", "GET"}, {"/api/v1/ops-alerts/:id", "GET"}}
+		for _, role := range []string{"implementation_operator", "site_admin", "viewer"} {
+			if err = attachDeviceRole(tx, role, []migrationModels.SysMenu{page, detail}); err != nil {
+				return err
+			}
+			for _, policy := range readPolicies {
+				if err = tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&deviceCasbinRule{Ptype: "p", V0: role, V1: policy[0], V2: policy[1]}).Error; err != nil {
+					return err
+				}
+			}
+		}
+		for _, role := range []string{"implementation_operator", "site_admin"} {
+			if err = attachDeviceRole(tx, role, []migrationModels.SysMenu{refresh, ack, recover}); err != nil {
+				return err
+			}
+			for _, policy := range [][2]string{{"/api/v1/ops-alerts/evaluate", "POST"}, {"/api/v1/ops-alerts/:id/acknowledge", "POST"}, {"/api/v1/ops-alerts/:id/recover", "POST"}} {
+				if err = tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&deviceCasbinRule{Ptype: "p", V0: role, V1: policy[0], V2: policy[1]}).Error; err != nil {
+					return err
+				}
+			}
+		}
+		if err = rebuildSenseMenuPaths(tx, root.MenuId, "/0"); err != nil {
+			return err
+		}
+		return tx.Create(&common.Migration{Version: version}).Error
+	})
+}

Sense/server/cmd/migrate/migration/version/2026082816000_ops_alert_test.go

@@ -0,0 +1,39 @@
+package version
+
+import (
+	"git.ilapage.cn/ila/yovision/Sense/server/app/sense/ops_alert"
+	migrationModels "git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration/models"
+	common "git.ilapage.cn/ila/yovision/Sense/server/common/models"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"testing"
+)
+
+func TestSenseOpsAlertMigrationRBAC(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = db.AutoMigrate(&migrationModels.SysRole{}, &migrationModels.SysMenu{}, &deviceCasbinRule{}, &common.Migration{}); err != nil {
+		t.Fatal(err)
+	}
+	for _, role := range []string{"implementation_operator", "site_admin", "viewer"} {
+		if err = db.Create(&migrationModels.SysRole{RoleName: role, RoleKey: role, Status: "2"}).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	const version = "2026082816000_ops_alert.go"
+	if err = migrateSenseOpsAlert(db, version); err != nil {
+		t.Fatal(err)
+	}
+	var pages, viewerWrites, operatorWrites int64
+	db.Model(&migrationModels.SysMenu{}).Where("permission = ?", "sense:ops-alert:list").Count(&pages)
+	db.Model(&deviceCasbinRule{}).Where("v0 = ? AND v1 LIKE ? AND v2 = ?", "viewer", "/api/v1/ops-alerts%", "POST").Count(&viewerWrites)
+	db.Model(&deviceCasbinRule{}).Where("v0 = ? AND v1 LIKE ? AND v2 = ?", "implementation_operator", "/api/v1/ops-alerts%", "POST").Count(&operatorWrites)
+	if pages != 1 || viewerWrites != 0 || operatorWrites != 3 {
+		t.Fatalf("unexpected RBAC page=%d viewerWrites=%d operatorWrites=%d", pages, viewerWrites, operatorWrites)
+	}
+	if !db.Migrator().HasTable(&ops_alert.Alert{}) || !db.Migrator().HasTable(&ops_alert.Transition{}) {
+		t.Fatal("ops alert tables missing")
+	}
+}