Compare commits

...
Author SHA1 Message Date
admin ea76db579c Merge pull request 'release: 将已验收 dev 基线合入 main' (#125) from dev into main
Reviewed-on: #125
2026-08-28 22:11:49 +08:00
ila 4a605f6482 Merge pull request #123 from feat/sense-79-ops-alerts
feat: 实现 Sense 运维告警隔离 (#79)
2026-08-28 18:25:33 +08:00
QiuSW d61e6d5ee1 feat: 实现 Sense 运维告警隔离 (#79) 2026-08-28 18:25:07 +08:00
ila 7274bd42f5 Merge PR #122: Sense 可靠投递状态机
Implements #78. Issue remains open for user acceptance.
2026-08-28 17:23:01 +08:00
QiuSW 02a5af5e3b feat: 建立 Sense 可靠投递状态机 (#78) 2026-08-28 17:22:44 +08:00
ila e06904272a Merge PR #121: [SEN] MediaMTX 分片分配与故障范围可观察
实现工单 #77;保持工单待人工验收。
2026-08-28 16:10:21 +08:00
35 changed files with 2708 additions and 0 deletions
@@ -0,0 +1,22 @@
package router
import (
"github.com/gin-gonic/gin"
jwt "github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/ops_alert"
"git.ilapage.cn/ila/yovision/Sense/server/common/actions"
"git.ilapage.cn/ila/yovision/Sense/server/common/middleware"
)
func init() { routerCheckRole = append(routerCheckRole, registerSenseOpsAlertRouter) }
func registerSenseOpsAlertRouter(v1 *gin.RouterGroup, auth *jwt.GinJWTMiddleware) {
api := &ops_alert.API{}
r := v1.Group("/ops-alerts").Use(auth.MiddlewareFunc()).Use(middleware.AuthCheckRole()).Use(actions.PermissionAction())
r.GET("", api.List)
r.GET("/:id", api.Get)
r.POST("/evaluate", api.Evaluate)
r.POST("/:id/acknowledge", api.Acknowledge)
r.POST("/:id/recover", api.Recover)
}
@@ -0,0 +1,32 @@
package router
import (
"github.com/gin-gonic/gin"
jwt "github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth"
"net/http"
"testing"
)
func TestSenseOpsAlertRoutes(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
registerSenseOpsAlertRouter(engine.Group("/api/v1"), &jwt.GinJWTMiddleware{})
wanted := map[string]bool{
http.MethodGet + " /api/v1/ops-alerts": false,
http.MethodGet + " /api/v1/ops-alerts/:id": false,
http.MethodPost + " /api/v1/ops-alerts/evaluate": false,
http.MethodPost + " /api/v1/ops-alerts/:id/acknowledge": false,
http.MethodPost + " /api/v1/ops-alerts/:id/recover": false,
}
for _, route := range engine.Routes() {
key := route.Method + " " + route.Path
if _, ok := wanted[key]; ok {
wanted[key] = true
}
}
for route, found := range wanted {
if !found {
t.Fatalf("route not registered: %s", route)
}
}
}
@@ -0,0 +1,19 @@
package router
import (
"github.com/gin-gonic/gin"
jwt "github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/outbox"
"git.ilapage.cn/ila/yovision/Sense/server/common/actions"
"git.ilapage.cn/ila/yovision/Sense/server/common/middleware"
)
func init() { routerCheckRole = append(routerCheckRole, registerSenseOutboxRouter) }
func registerSenseOutboxRouter(v1 *gin.RouterGroup, auth *jwt.GinJWTMiddleware) {
api := &outbox.API{}
r := v1.Group("/outbox").Use(auth.MiddlewareFunc()).Use(middleware.AuthCheckRole()).Use(actions.PermissionAction())
r.GET("", api.List)
r.GET("/:id", api.Get)
r.POST("/:id/requeue", api.Requeue)
}
@@ -0,0 +1,26 @@
package router
import (
"github.com/gin-gonic/gin"
jwt "github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth"
"net/http"
"testing"
)
func TestSenseOutboxRoutes(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
registerSenseOutboxRouter(engine.Group("/api/v1"), &jwt.GinJWTMiddleware{})
wanted := map[string]bool{http.MethodGet + " /api/v1/outbox": false, http.MethodGet + " /api/v1/outbox/:id": false, http.MethodPost + " /api/v1/outbox/:id/requeue": false}
for _, route := range engine.Routes() {
key := route.Method + " " + route.Path
if _, ok := wanted[key]; ok {
wanted[key] = true
}
}
for route, found := range wanted {
if !found {
t.Fatalf("route not registered: %s", route)
}
}
}
@@ -0,0 +1,29 @@
package local_event
import (
"context"
"encoding/json"
"fmt"
"time"
"gorm.io/gorm"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/outbox"
)
// CreateWithOutbox commits the local candidate and its internal delivery
// record atomically. The payload remains Sense-internal and is not a Bell or
// Brain contract.
func CreateWithOutbox(ctx context.Context, db *gorm.DB, candidate EventCandidate, payload map[string]interface{}, now time.Time) error {
encoded, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("encode local event outbox payload: %w", err)
}
return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Create(&candidate).Error; err != nil {
return fmt.Errorf("create local event candidate: %w", err)
}
_, err = outbox.Enqueue(tx, outbox.EnqueueInput{InternalType: "local_event_candidate", BusinessRef: candidate.ID, IdempotencyKey: "local-event:" + candidate.ID + ":v1", PayloadJSON: encoded}, now)
return err
})
}
@@ -0,0 +1,46 @@
package local_event
import (
"context"
"testing"
"time"
"github.com/google/uuid"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/outbox"
)
func TestCreateWithOutboxCommitsAndRollsBackAtomically(t *testing.T) {
db, err := gorm.Open(sqlite.Open("file:"+uuid.NewString()+"?mode=memory&cache=shared"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
sqlDB, _ := db.DB()
sqlDB.SetMaxOpenConns(1)
if err = db.AutoMigrate(&EventCandidate{}, &outbox.Message{}, &outbox.DeliveryRecord{}, &outbox.Attempt{}); err != nil {
t.Fatal(err)
}
now := time.Date(2026, 8, 28, 10, 0, 0, 0, time.UTC)
candidate := EventCandidate{ID: uuid.NewString(), OccurredAt: now, SourceRef: "SEN-CAM-01", RuleRef: "rule-1", RuleName: "区域闯入", CandidateState: CandidateStateCandidate, EvidenceState: EvidenceStatePending, RetainUntil: now.Add(24 * time.Hour)}
if err = CreateWithOutbox(context.Background(), db, candidate, map[string]interface{}{"eventId": candidate.ID}, now); err != nil {
t.Fatal(err)
}
var candidates, messages int64
db.Model(&EventCandidate{}).Count(&candidates)
db.Model(&outbox.Message{}).Count(&messages)
if candidates != 1 || messages != 1 {
t.Fatalf("candidates=%d messages=%d", candidates, messages)
}
duplicate := candidate
duplicate.ID = candidate.ID
if err = CreateWithOutbox(context.Background(), db, duplicate, map[string]interface{}{"eventId": duplicate.ID}, now); err == nil {
t.Fatal("expected duplicate transaction failure")
}
db.Model(&EventCandidate{}).Count(&candidates)
db.Model(&outbox.Message{}).Count(&messages)
if candidates != 1 || messages != 1 {
t.Fatalf("atomic rollback failed candidates=%d messages=%d", candidates, messages)
}
}
+121
View File
@@ -0,0 +1,121 @@
package ops_alert
import (
"errors"
"git.ilapage.cn/ila/yovision/Sense/server/common"
"github.com/gin-gonic/gin"
"github.com/gin-gonic/gin/binding"
"github.com/go-admin-team/go-admin-core/sdk/api"
"github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth/user"
coreService "github.com/go-admin-team/go-admin-core/sdk/service"
"net/http"
"time"
)
const auditSuccess, auditFailure = "1", "2"
type API struct{ api.Api }
func (e *API) service(c *gin.Context) (*Service, error) {
base := coreService.Service{}
if err := e.MakeContext(c).MakeOrm().MakeService(&base).Errors; err != nil {
return nil, err
}
return NewService(base.Orm), nil
}
func (e *API) List(c *gin.Context) {
service, err := e.service(c)
if err != nil {
e.writeError(err)
return
}
request := PageRequest{}
if err = e.MakeContext(c).Bind(&request).Errors; err != nil {
e.audit(c, service, "List", auditFailure, "告警查询条件格式不正确")
e.Error(http.StatusBadRequest, err, "查询条件格式不正确")
return
}
response, err := service.List(request)
if err != nil {
e.audit(c, service, "List", auditFailure, "告警列表查询失败")
e.writeError(err)
return
}
e.audit(c, service, "List", auditSuccess, "读取运维告警列表")
e.OK(response, "查询成功")
}
func (e *API) Get(c *gin.Context) {
service, err := e.service(c)
if err != nil {
e.writeError(err)
return
}
response, err := service.Get(c.Param("id"))
if err != nil {
e.audit(c, service, "Get", auditFailure, "告警详情查询失败")
e.writeError(err)
return
}
e.audit(c, service, "Get", auditSuccess, "读取运维告警详情 "+response.Alert.ID)
e.OK(response, "查询成功")
}
func (e *API) Evaluate(c *gin.Context) {
service, err := e.service(c)
if err != nil {
e.writeError(err)
return
}
response, err := service.EvaluateSources(c.Request.Context())
if err != nil {
e.audit(c, service, "Evaluate", auditFailure, "健康事实刷新失败")
e.writeError(err)
return
}
e.audit(c, service, "Evaluate", auditSuccess, "刷新 Sense 内部健康事实")
e.OK(response, "状态已刷新")
}
func (e *API) Acknowledge(c *gin.Context) { e.action(c, "Acknowledge") }
func (e *API) Recover(c *gin.Context) { e.action(c, "Recover") }
func (e *API) action(c *gin.Context, action string) {
service, err := e.service(c)
if err != nil {
e.writeError(err)
return
}
request := ActionRequest{}
if err = e.MakeContext(c).Bind(&request, binding.JSON).Errors; err != nil {
e.audit(c, service, action, auditFailure, "告警操作请求格式不正确")
e.Error(http.StatusBadRequest, err, "请求格式不正确")
return
}
var item Alert
if action == "Acknowledge" {
item, err = service.Acknowledge(c.Request.Context(), c.Param("id"), request, user.GetUserId(c))
} else {
item, err = service.Recover(c.Request.Context(), c.Param("id"), request, user.GetUserId(c))
}
if err != nil {
e.audit(c, service, action, auditFailure, "运维告警状态操作被拒绝")
e.writeError(err)
return
}
e.audit(c, service, action, auditSuccess, action+" "+item.ID)
e.OK(item, "操作成功")
}
func (e *API) audit(c *gin.Context, service *Service, action, status, remark string) {
if err := WriteAudit(service.DB, Audit{Action: action, Method: c.Request.Method, Status: status, Username: user.GetUserName(c), UserID: user.GetUserId(c), ClientIP: common.GetClientIP(c), Route: c.FullPath(), Remark: remark, At: time.Now()}); err != nil {
api.GetRequestLogger(c).Errorf("ops alert audit failed: %s", err.Error())
}
}
func (e *API) writeError(err error) {
switch {
case errors.Is(err, ErrInvalidFilter), errors.Is(err, ErrInvalidReason):
e.Error(http.StatusBadRequest, err, err.Error())
case errors.Is(err, ErrVersionConflict), errors.Is(err, ErrInvalidTransition), errors.Is(err, ErrRecoveryWindowOpen):
e.Error(http.StatusConflict, err, err.Error())
case errors.Is(err, ErrAlertNotFound):
e.Error(http.StatusNotFound, err, err.Error())
default:
e.Error(http.StatusInternalServerError, err, "运维告警操作失败")
}
}
+19
View File
@@ -0,0 +1,19 @@
package ops_alert
import (
adminModels "git.ilapage.cn/ila/yovision/Sense/server/app/admin/models"
"gorm.io/gorm"
"time"
)
type Audit struct {
Action, Method, Status, Username, ClientIP, Route, Remark string
UserID int
At time.Time
}
func WriteAudit(db *gorm.DB, input Audit) error {
model := adminModels.SysOperaLog{Title: "运维告警", BusinessType: "other", Method: "ops_alert.API." + input.Action, RequestMethod: input.Method, OperatorType: "1", OperName: input.Username, OperUrl: input.Route, OperIp: input.ClientIP, Status: input.Status, OperTime: input.At.UTC(), Remark: input.Remark, CreatedAt: input.At.UTC(), UpdatedAt: input.At.UTC()}
model.CreateBy, model.UpdateBy = input.UserID, input.UserID
return db.Create(&model).Error
}
+37
View File
@@ -0,0 +1,37 @@
package ops_alert
import commonDto "git.ilapage.cn/ila/yovision/Sense/server/common/dto"
type PageRequest struct {
commonDto.Pagination `search:"-"`
AlertType string `form:"alertType"`
State string `form:"state"`
Keyword string `form:"keyword"`
}
type Summary struct {
Active int64 `json:"active"`
Unacknowledged int64 `json:"unacknowledged"`
Recovering int64 `json:"recovering"`
}
type PageResponse struct {
Summary Summary `json:"summary"`
List []Alert `json:"list"`
Count int64 `json:"count"`
}
type DetailResponse struct {
Alert Alert `json:"alert"`
Transitions []Transition `json:"transitions"`
}
type ActionRequest struct {
ExpectedVersion int64 `json:"expectedVersion" binding:"required,min=1"`
Reason string `json:"reason" binding:"required"`
}
type EvaluateResponse struct {
Observed int `json:"observed"`
Abnormal int `json:"abnormal"`
}
@@ -0,0 +1,62 @@
package ops_alert
import "time"
const (
StateUnacknowledged = "unacknowledged"
StateAcknowledged = "acknowledged"
StateRecovering = "recovering"
StateRecovered = "recovered"
TypeDeviceOffline = "device_offline"
TypeAuthentication = "authentication_failed"
TypeClockDrift = "clock_drift"
TypeReconciliation = "reconciliation_failed"
TypeMediaShard = "media_shard_failed"
TypeControlTunnel = "control_tunnel_failed"
)
type Alert struct {
ID string `gorm:"size:36;primaryKey" json:"id"`
Fingerprint string `gorm:"size:255;not null;uniqueIndex" json:"fingerprint"`
AlertType string `gorm:"size:32;not null;index" json:"alertType"`
Severity string `gorm:"size:16;not null;index" json:"severity"`
ObjectType string `gorm:"size:32;not null;index" json:"objectType"`
ObjectID string `gorm:"size:96;not null;index" json:"objectId"`
ObjectName string `gorm:"size:128;not null" json:"objectName"`
Location string `gorm:"size:255;not null;default:''" json:"location"`
State string `gorm:"size:24;not null;index" json:"state"`
Title string `gorm:"size:160;not null" json:"title"`
Detail string `gorm:"size:512;not null" json:"detail"`
NextAction string `gorm:"size:255;not null" json:"nextAction"`
LastSourceVersion string `gorm:"size:128;not null;default:''" json:"-"`
OccurrenceCount int `gorm:"not null;default:1" json:"occurrenceCount"`
Cycle int `gorm:"not null;default:1" json:"cycle"`
FirstSeenAt time.Time `gorm:"not null;index" json:"firstSeenAt"`
LastSeenAt time.Time `gorm:"not null;index" json:"lastSeenAt"`
HealthySince *time.Time `json:"healthySince,omitempty"`
AcknowledgedAt *time.Time `json:"acknowledgedAt,omitempty"`
AcknowledgedBy int `gorm:"not null;default:0" json:"acknowledgedBy,omitempty"`
RecoveredAt *time.Time `json:"recoveredAt,omitempty"`
RecoveredBy int `gorm:"not null;default:0" json:"recoveredBy,omitempty"`
OperationalOnly bool `gorm:"not null;default:true" json:"operationalOnly"`
Version int64 `gorm:"not null;default:1" json:"version"`
CreatedAt time.Time `gorm:"not null" json:"createdAt"`
UpdatedAt time.Time `gorm:"not null" json:"updatedAt"`
}
func (Alert) TableName() string { return "sense_ops_alerts" }
type Transition struct {
ID string `gorm:"size:36;primaryKey" json:"id"`
AlertID string `gorm:"size:36;not null;index" json:"alertId"`
Cycle int `gorm:"not null" json:"cycle"`
Action string `gorm:"size:32;not null;index" json:"action"`
FromState string `gorm:"size:24;not null;default:''" json:"fromState"`
ToState string `gorm:"size:24;not null" json:"toState"`
Reason string `gorm:"size:512;not null;default:''" json:"reason"`
ActorUserID int `gorm:"not null;default:0" json:"actorUserId"`
CreatedAt time.Time `gorm:"not null;index" json:"createdAt"`
}
func (Transition) TableName() string { return "sense_ops_alert_transitions" }
+326
View File
@@ -0,0 +1,326 @@
package ops_alert
import (
"context"
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/clause"
deviceModels "git.ilapage.cn/ila/yovision/Sense/server/app/sense/device/models"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/edge_node"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media_shard"
)
var (
ErrAlertNotFound = errors.New("运维告警不存在")
ErrInvalidFilter = errors.New("告警筛选条件不正确")
ErrInvalidReason = errors.New("处理说明需为 6 至 256 个字符")
ErrInvalidTransition = errors.New("当前告警状态不允许此操作")
ErrVersionConflict = errors.New("告警状态已变化,请刷新后重试")
ErrRecoveryWindowOpen = errors.New("健康恢复观察窗口尚未结束")
)
const defaultRecoveryWindow = 5 * time.Minute
type Service struct {
DB *gorm.DB
Now func() time.Time
RecoveryWindow time.Duration
}
func NewService(db *gorm.DB) *Service {
return &Service{DB: db, Now: time.Now, RecoveryWindow: defaultRecoveryWindow}
}
func (s *Service) List(request PageRequest) (PageResponse, error) {
if !validOptional(request.AlertType, alertTypes()) || !validOptional(request.State, states()) {
return PageResponse{}, ErrInvalidFilter
}
query := s.DB.Model(&Alert{})
if request.AlertType != "" {
query = query.Where("alert_type = ?", request.AlertType)
}
if request.State != "" {
query = query.Where("state = ?", request.State)
}
if keyword := strings.TrimSpace(request.Keyword); keyword != "" {
like := "%" + keyword + "%"
query = query.Where("object_name LIKE ? OR location LIKE ? OR title LIKE ? OR detail LIKE ?", like, like, like, like)
}
response := PageResponse{}
if err := query.Count(&response.Count).Error; err != nil {
return response, err
}
if err := query.Order("CASE WHEN state = 'unacknowledged' THEN 0 WHEN state = 'acknowledged' THEN 1 WHEN state = 'recovering' THEN 2 ELSE 3 END, last_seen_at DESC").Offset((request.GetPageIndex() - 1) * request.GetPageSize()).Limit(request.GetPageSize()).Find(&response.List).Error; err != nil {
return response, err
}
active := []string{StateUnacknowledged, StateAcknowledged, StateRecovering}
if err := s.DB.Model(&Alert{}).Where("state IN ?", active).Count(&response.Summary.Active).Error; err != nil {
return response, err
}
if err := s.DB.Model(&Alert{}).Where("state = ?", StateUnacknowledged).Count(&response.Summary.Unacknowledged).Error; err != nil {
return response, err
}
if err := s.DB.Model(&Alert{}).Where("state = ?", StateRecovering).Count(&response.Summary.Recovering).Error; err != nil {
return response, err
}
return response, nil
}
func (s *Service) Get(id string) (DetailResponse, error) {
response := DetailResponse{Transitions: []Transition{}}
if err := s.DB.First(&response.Alert, "id = ?", id).Error; errors.Is(err, gorm.ErrRecordNotFound) {
return response, ErrAlertNotFound
} else if err != nil {
return response, err
}
if err := s.DB.Where("alert_id = ?", id).Order("created_at DESC").Find(&response.Transitions).Error; err != nil {
return response, err
}
return response, nil
}
func (s *Service) Acknowledge(ctx context.Context, id string, request ActionRequest, actor int) (Alert, error) {
return s.transition(ctx, id, request, actor, StateUnacknowledged, StateAcknowledged, "acknowledge")
}
func (s *Service) Recover(ctx context.Context, id string, request ActionRequest, actor int) (Alert, error) {
return s.transition(ctx, id, request, actor, StateRecovering, StateRecovered, "recover")
}
func (s *Service) transition(ctx context.Context, id string, request ActionRequest, actor int, expectedState, targetState, action string) (Alert, error) {
reason := strings.TrimSpace(request.Reason)
if utf8.RuneCountInString(reason) < 6 || utf8.RuneCountInString(reason) > 256 {
return Alert{}, ErrInvalidReason
}
if request.ExpectedVersion < 1 {
return Alert{}, ErrVersionConflict
}
now := s.Now().UTC()
result := Alert{}
err := s.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var current Alert
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&current, "id = ?", id).Error; errors.Is(err, gorm.ErrRecordNotFound) {
return ErrAlertNotFound
} else if err != nil {
return err
}
if current.Version != request.ExpectedVersion {
return ErrVersionConflict
}
if current.State != expectedState {
return ErrInvalidTransition
}
if action == "recover" && (current.HealthySince == nil || now.Sub(current.HealthySince.UTC()) < s.RecoveryWindow) {
return ErrRecoveryWindowOpen
}
updates := map[string]any{"state": targetState, "version": current.Version + 1, "updated_at": now}
if action == "acknowledge" {
updates["acknowledged_at"], updates["acknowledged_by"] = now, actor
} else {
updates["recovered_at"], updates["recovered_by"] = now, actor
}
write := tx.Model(&Alert{}).Where("id = ? AND version = ? AND state = ?", id, current.Version, expectedState).Updates(updates)
if write.Error != nil {
return write.Error
}
if write.RowsAffected != 1 {
return ErrVersionConflict
}
if err := tx.Create(&Transition{ID: uuid.NewString(), AlertID: id, Cycle: current.Cycle, Action: action, FromState: expectedState, ToState: targetState, Reason: reason, ActorUserID: actor, CreatedAt: now}).Error; err != nil {
return err
}
return tx.First(&result, "id = ?", id).Error
})
return result, err
}
type observation struct {
Fingerprint, AlertType, Severity, ObjectType, ObjectID, ObjectName, Location, Title, Detail, NextAction, SourceVersion string
Abnormal bool
}
func (s *Service) EvaluateSources(ctx context.Context) (EvaluateResponse, error) {
items, err := s.sourceObservations(ctx)
if err != nil {
return EvaluateResponse{}, err
}
response := EvaluateResponse{Observed: len(items)}
for _, item := range items {
if item.Abnormal {
response.Abnormal++
}
if err = s.observe(ctx, item); err != nil {
return response, err
}
}
return response, nil
}
func (s *Service) observe(ctx context.Context, item observation) error {
now := s.Now().UTC()
return s.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var current Alert
err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("fingerprint = ?", item.Fingerprint).First(&current).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
if !item.Abnormal {
return nil
}
current = Alert{ID: uuid.NewString(), Fingerprint: item.Fingerprint, AlertType: item.AlertType, Severity: item.Severity, ObjectType: item.ObjectType, ObjectID: item.ObjectID, ObjectName: item.ObjectName, Location: item.Location, State: StateUnacknowledged, Title: item.Title, Detail: item.Detail, NextAction: item.NextAction, LastSourceVersion: item.SourceVersion, OccurrenceCount: 1, Cycle: 1, FirstSeenAt: now, LastSeenAt: now, OperationalOnly: true, Version: 1, CreatedAt: now, UpdatedAt: now}
if err = tx.Create(&current).Error; err != nil {
return err
}
return tx.Create(&Transition{ID: uuid.NewString(), AlertID: current.ID, Cycle: 1, Action: "detected", ToState: StateUnacknowledged, Reason: "健康事实首次满足告警规则", CreatedAt: now}).Error
}
if err != nil {
return err
}
if item.Abnormal {
updates := map[string]any{"severity": item.Severity, "object_name": item.ObjectName, "location": item.Location, "title": item.Title, "detail": item.Detail, "next_action": item.NextAction, "last_seen_at": now, "updated_at": now, "healthy_since": nil}
if item.SourceVersion != current.LastSourceVersion {
updates["last_source_version"], updates["occurrence_count"] = item.SourceVersion, current.OccurrenceCount+1
}
if current.State == StateRecovered {
updates["state"], updates["cycle"], updates["first_seen_at"], updates["occurrence_count"], updates["acknowledged_at"], updates["acknowledged_by"], updates["recovered_at"], updates["recovered_by"] = StateUnacknowledged, current.Cycle+1, now, 1, nil, 0, nil, 0
if err = tx.Create(&Transition{ID: uuid.NewString(), AlertID: current.ID, Cycle: current.Cycle + 1, Action: "reopened", FromState: StateRecovered, ToState: StateUnacknowledged, Reason: "健康事实再次异常", CreatedAt: now}).Error; err != nil {
return err
}
} else if current.State == StateRecovering {
target := StateUnacknowledged
if current.AcknowledgedAt != nil {
target = StateAcknowledged
}
updates["state"] = target
if err = tx.Create(&Transition{ID: uuid.NewString(), AlertID: current.ID, Cycle: current.Cycle, Action: "relapsed", FromState: StateRecovering, ToState: target, Reason: "恢复观察期间再次异常", CreatedAt: now}).Error; err != nil {
return err
}
}
updates["version"] = current.Version + 1
return tx.Model(&Alert{}).Where("id = ? AND version = ?", current.ID, current.Version).Updates(updates).Error
}
if current.State == StateUnacknowledged || current.State == StateAcknowledged {
if err = tx.Create(&Transition{ID: uuid.NewString(), AlertID: current.ID, Cycle: current.Cycle, Action: "health_restored", FromState: current.State, ToState: StateRecovering, Reason: "健康事实已恢复,进入稳定观察窗口", CreatedAt: now}).Error; err != nil {
return err
}
return tx.Model(&Alert{}).Where("id = ? AND version = ?", current.ID, current.Version).Updates(map[string]any{"state": StateRecovering, "healthy_since": now, "version": current.Version + 1, "updated_at": now}).Error
}
return nil
})
}
type admissionProjection struct {
DeviceID, Status, Detail string
CheckedAt time.Time
}
func (admissionProjection) TableName() string { return "sense_admission_results" }
func (s *Service) sourceObservations(ctx context.Context) ([]observation, error) {
now := s.Now().UTC()
result := []observation{}
var devices []deviceModels.Device
if err := s.DB.WithContext(ctx).Find(&devices).Error; err != nil {
return nil, err
}
var admissions []admissionProjection
if err := s.DB.WithContext(ctx).Find(&admissions).Error; err != nil && !isMissingTable(err) {
return nil, err
}
byDevice := map[string]admissionProjection{}
for _, item := range admissions {
byDevice[item.DeviceID] = item
}
for _, device := range devices {
if device.Status == deviceModels.StatusDisabled {
continue
}
state, detail, observed := device.AdapterStatus, "尚未获得设备接入结果", device.UpdatedAt
if item, ok := byDevice[device.ID]; ok {
state, detail, observed = item.Status, item.Detail, item.CheckedAt
}
text := strings.ToLower(state + " " + detail)
version := fmt.Sprintf("%d:%d", device.Version, observed.UnixNano())
base := observation{ObjectType: "device", ObjectID: device.ID, ObjectName: device.Name, Location: device.Location, SourceVersion: version}
result = append(result,
withRule(base, TypeDeviceOffline, "high", "设备离线", detail, "检查设备供电、网络和地址后重新发现", containsAny(text, "offline", "unreachable", "timeout", "离线", "无法连接", "超时")),
withRule(base, TypeAuthentication, "high", "设备认证失败", detail, "更新设备凭据后重新执行接入验证", containsAny(text, "authentication_failed", "unauthorized", "auth", "认证", "凭据")),
withRule(base, TypeClockDrift, "medium", "设备时间漂移", detail, "校准设备时间后重新执行接入验证", containsAny(text, "clock_skew", "clock_drift", "time drift", "时间漂移", "时钟")),
)
}
var routes []media.Route
if err := s.DB.WithContext(ctx).Find(&routes).Error; err != nil && !isMissingTable(err) {
return nil, err
}
for _, route := range routes {
converged := (route.Desired == media.DesiredRunning && (route.Actual == "ready" || route.Actual == "waiting")) || (route.Desired == media.DesiredStopped && route.Actual == "stopped")
base := observation{ObjectType: "media_route", ObjectID: route.ID, ObjectName: route.Path, SourceVersion: fmt.Sprintf("%d", route.Version)}
result = append(result, withRule(base, TypeReconciliation, "medium", "媒体状态未收敛", route.Detail, "在运维中心核对期望态并执行受控重试", !converged))
}
var shards []media_shard.Shard
if err := s.DB.WithContext(ctx).Find(&shards).Error; err != nil && !isMissingTable(err) {
return nil, err
}
for _, shard := range shards {
if shard.Status == media_shard.StatusDisabled {
continue
}
stale := shard.LastProbeAt == nil || now.Sub(shard.LastProbeAt.UTC()) > 30*time.Second
base := observation{ObjectType: "media_shard", ObjectID: shard.ID, ObjectName: shard.Name, SourceVersion: fmt.Sprintf("%d:%d", shard.ConfigVersion, timeValue(shard.LastProbeAt))}
result = append(result, withRule(base, TypeMediaShard, "high", "媒体分片异常", shard.Detail, "检查 MediaMTX 进程和 Control API 后重新探测", shard.Status == media_shard.StatusFailed || stale))
}
var nodes []edge_node.Node
if err := s.DB.WithContext(ctx).Find(&nodes).Error; err != nil && !isMissingTable(err) {
return nil, err
}
for _, node := range nodes {
base := observation{ObjectType: "edge_node", ObjectID: node.ID, ObjectName: node.Name, Location: node.Location, SourceVersion: fmt.Sprintf("%d:%d", node.ProjectionVersion, node.LastHeartbeatAt.UnixNano())}
result = append(result,
withRule(base, TypeDeviceOffline, "high", "边缘节点离线", "心跳超过 90 秒未更新", "检查节点进程、网络和机器身份后等待心跳恢复", now.Sub(node.LastHeartbeatAt.UTC()) > edge_node.HeartbeatTimeout),
withRule(base, TypeControlTunnel, "high", "控制隧道异常", node.ControlTunnelDetail, "检查节点到 Sense 的控制通道,不影响业务预警数据", node.ControlTunnelStatus != edge_node.ChannelReady),
)
}
return result, nil
}
func withRule(base observation, typ, severity, title, detail, next string, abnormal bool) observation {
base.Fingerprint = typ + ":" + base.ObjectType + ":" + base.ObjectID
base.AlertType = typ
base.Severity = severity
base.Title = title
base.Detail = detail
base.NextAction = next
base.Abnormal = abnormal
return base
}
func containsAny(value string, needles ...string) bool {
for _, needle := range needles {
if strings.Contains(value, needle) {
return true
}
}
return false
}
func isMissingTable(err error) bool {
text := strings.ToLower(err.Error())
return strings.Contains(text, "no such table") || strings.Contains(text, "does not exist")
}
func timeValue(value *time.Time) int64 {
if value == nil {
return 0
}
return value.UnixNano()
}
func validOptional(value string, allowed map[string]bool) bool { return value == "" || allowed[value] }
func states() map[string]bool {
return map[string]bool{StateUnacknowledged: true, StateAcknowledged: true, StateRecovering: true, StateRecovered: true}
}
func alertTypes() map[string]bool {
return map[string]bool{TypeDeviceOffline: true, TypeAuthentication: true, TypeClockDrift: true, TypeReconciliation: true, TypeMediaShard: true, TypeControlTunnel: true}
}
@@ -0,0 +1,198 @@
package ops_alert
import (
"context"
"errors"
"testing"
"time"
"github.com/google/uuid"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
adminModels "git.ilapage.cn/ila/yovision/Sense/server/app/admin/models"
deviceModels "git.ilapage.cn/ila/yovision/Sense/server/app/sense/device/models"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/edge_node"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/media_shard"
)
func opsAlertTestService(t *testing.T) (*Service, *gorm.DB, time.Time) {
t.Helper()
db, err := gorm.Open(sqlite.Open("file:"+uuid.NewString()+"?mode=memory&cache=shared"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err = db.AutoMigrate(&Alert{}, &Transition{}, &deviceModels.Device{}, &admissionProjection{}, &media.Route{}, &media_shard.Shard{}, &edge_node.Node{}, &adminModels.SysOperaLog{}); err != nil {
t.Fatal(err)
}
now := time.Date(2026, 8, 28, 8, 0, 0, 0, time.UTC)
service := NewService(db)
service.Now = func() time.Time { return now }
return service, db, now
}
func TestEvaluateSourcesGeneratesSixTypesAndDeduplicates(t *testing.T) {
service, db, now := opsAlertTestService(t)
devices := []deviceModels.Device{
{ID: "offline", Name: "东门摄像机", Location: "东门", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 1},
{ID: "auth", Name: "仓库摄像机", Location: "仓库", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 2},
{ID: "clock", Name: "南门摄像机", Location: "南门", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 3},
}
if err := db.Create(&devices).Error; err != nil {
t.Fatal(err)
}
results := []admissionProjection{
{DeviceID: "offline", Status: "device_offline", Detail: "设备离线,无法连接", CheckedAt: now},
{DeviceID: "auth", Status: "authentication_failed", Detail: "设备认证失败", CheckedAt: now},
{DeviceID: "clock", Status: "clock_skew", Detail: "设备时间漂移", CheckedAt: now},
}
if err := db.Create(&results).Error; err != nil {
t.Fatal(err)
}
if err := db.Create(&media.Route{ID: "route-1", DeviceID: "auth", ProfileToken: "main", Path: "sense_auth", Desired: "running", Actual: "apply_failed", Detail: "配置未收敛", Version: 4, UpdatedAt: now}).Error; err != nil {
t.Fatal(err)
}
probe := now
if err := db.Create(&media_shard.Shard{ID: "shard-1", Name: "媒体服务 A", Mode: "local", ControlAPI: "http://127.0.0.1:9997", Capacity: 16, Status: media_shard.StatusFailed, Detail: "Control API 不可用", LastProbeAt: &probe, ConfigVersion: 1, CreatedAt: now, UpdatedAt: now}).Error; err != nil {
t.Fatal(err)
}
if err := db.Create(&edge_node.Node{ID: "node-1", Name: "主楼边缘节点", Location: "机房", StartedAt: now.Add(-time.Hour), LastHeartbeatAt: now.Add(-10 * time.Second), LastCollectedAt: now.Add(-10 * time.Second), ControlTunnelStatus: edge_node.ChannelUnavailable, ControlTunnelDetail: "TLS 隧道断开", VideoPlaneStatus: edge_node.ChannelReady, ProjectionVersion: 7}).Error; err != nil {
t.Fatal(err)
}
response, err := service.EvaluateSources(context.Background())
if err != nil {
t.Fatal(err)
}
if response.Abnormal != 6 {
t.Fatalf("abnormal=%d want 6", response.Abnormal)
}
var alerts []Alert
if err = db.Find(&alerts).Error; err != nil {
t.Fatal(err)
}
if len(alerts) != 6 {
t.Fatalf("alerts=%d want 6", len(alerts))
}
types := map[string]bool{}
for _, item := range alerts {
types[item.AlertType] = true
if !item.OperationalOnly {
t.Fatal("alert crossed operational boundary")
}
}
for _, typ := range []string{TypeDeviceOffline, TypeAuthentication, TypeClockDrift, TypeReconciliation, TypeMediaShard, TypeControlTunnel} {
if !types[typ] {
t.Fatalf("missing type %s", typ)
}
}
if _, err = service.EvaluateSources(context.Background()); err != nil {
t.Fatal(err)
}
var count int64
db.Model(&Alert{}).Count(&count)
if count != 6 {
t.Fatalf("duplicate active alerts: %d", count)
}
var offline Alert
if err = db.First(&offline, "fingerprint = ?", TypeDeviceOffline+":device:offline").Error; err != nil {
t.Fatal(err)
}
if offline.OccurrenceCount != 1 {
t.Fatalf("unchanged source counted twice: %d", offline.OccurrenceCount)
}
}
func TestAcknowledgeRecoverAndReopenPreserveOneAlert(t *testing.T) {
service, db, now := opsAlertTestService(t)
device := deviceModels.Device{ID: "auth", Name: "仓库摄像机", Modality: "video", CapabilitiesJSON: "[]", Status: "pending", AdapterStatus: "verification_failed", Version: 1}
if err := db.Create(&device).Error; err != nil {
t.Fatal(err)
}
result := admissionProjection{DeviceID: "auth", Status: "authentication_failed", Detail: "认证失败", CheckedAt: now}
if err := db.Create(&result).Error; err != nil {
t.Fatal(err)
}
if _, err := service.EvaluateSources(context.Background()); err != nil {
t.Fatal(err)
}
var alert Alert
if err := db.First(&alert, "alert_type = ?", TypeAuthentication).Error; err != nil {
t.Fatal(err)
}
ack, err := service.Acknowledge(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "已安排现场人员更新设备凭据"}, 7)
if err != nil {
t.Fatal(err)
}
if ack.State != StateAcknowledged || ack.AcknowledgedBy != 7 {
t.Fatalf("unexpected acknowledgement: %+v", ack)
}
if _, err = service.Acknowledge(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "并发重复确认不应成功"}, 8); !errors.Is(err, ErrVersionConflict) {
t.Fatalf("want version conflict, got %v", err)
}
if err = db.Model(&admissionProjection{}).Where("device_id = ?", "auth").Updates(map[string]any{"status": "ready", "detail": "接入验证完成", "checked_at": now.Add(time.Minute)}).Error; err != nil {
t.Fatal(err)
}
service.Now = func() time.Time { return now.Add(time.Minute) }
if _, err = service.EvaluateSources(context.Background()); err != nil {
t.Fatal(err)
}
if err = db.First(&alert, "id = ?", alert.ID).Error; err != nil {
t.Fatal(err)
}
if alert.State != StateRecovering || alert.HealthySince == nil {
t.Fatalf("want recovering: %+v", alert)
}
if _, err = service.Recover(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "恢复窗口尚未结束,不能关闭"}, 7); !errors.Is(err, ErrRecoveryWindowOpen) {
t.Fatalf("want recovery window error, got %v", err)
}
service.Now = func() time.Time { return now.Add(7 * time.Minute) }
recovered, err := service.Recover(context.Background(), alert.ID, ActionRequest{ExpectedVersion: alert.Version, Reason: "设备已稳定在线超过恢复观察窗口"}, 7)
if err != nil {
t.Fatal(err)
}
if recovered.State != StateRecovered {
t.Fatalf("state=%s", recovered.State)
}
if _, err = service.EvaluateSources(context.Background()); err != nil {
t.Fatal(err)
}
if err = db.Model(&admissionProjection{}).Where("device_id = ?", "auth").Updates(map[string]any{"status": "authentication_failed", "detail": "认证再次失败", "checked_at": now.Add(8 * time.Minute)}).Error; err != nil {
t.Fatal(err)
}
service.Now = func() time.Time { return now.Add(8 * time.Minute) }
if _, err = service.EvaluateSources(context.Background()); err != nil {
t.Fatal(err)
}
var count int64
db.Model(&Alert{}).Where("fingerprint = ?", recovered.Fingerprint).Count(&count)
if count != 1 {
t.Fatalf("reopen duplicated alert: %d", count)
}
if err = db.First(&alert, "id = ?", alert.ID).Error; err != nil {
t.Fatal(err)
}
if alert.State != StateUnacknowledged || alert.Cycle != 2 {
t.Fatalf("unexpected reopened alert: %+v", alert)
}
var transitions int64
db.Model(&Transition{}).Where("alert_id = ?", alert.ID).Count(&transitions)
if transitions < 5 {
t.Fatalf("transitions=%d", transitions)
}
}
func TestAuditIsStoredOnlyInGoAdminOperationLog(t *testing.T) {
_, db, now := opsAlertTestService(t)
if err := WriteAudit(db, Audit{Action: "Acknowledge", Method: "POST", Status: "1", Username: "operator", UserID: 9, ClientIP: "127.0.0.1", Route: "/api/v1/ops-alerts/:id/acknowledge", Remark: "确认运维告警", At: now}); err != nil {
t.Fatal(err)
}
var log adminModels.SysOperaLog
if err := db.First(&log).Error; err != nil {
t.Fatal(err)
}
if log.Title != "运维告警" || log.CreateBy != 9 {
t.Fatalf("unexpected audit: %+v", log)
}
}
+103
View File
@@ -0,0 +1,103 @@
package outbox
import (
"errors"
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/gin-gonic/gin/binding"
"github.com/go-admin-team/go-admin-core/sdk/api"
"github.com/go-admin-team/go-admin-core/sdk/pkg/jwtauth/user"
coreService "github.com/go-admin-team/go-admin-core/sdk/service"
"git.ilapage.cn/ila/yovision/Sense/server/common"
)
type API struct{ api.Api }
func (e *API) service(c *gin.Context) (*Service, error) {
base := coreService.Service{}
if err := e.MakeContext(c).MakeOrm().MakeService(&base).Errors; err != nil {
return nil, err
}
return NewService(base.Orm), nil
}
func (e *API) List(c *gin.Context) {
service, err := e.service(c)
if err != nil {
e.writeError(err)
return
}
request := PageRequest{}
if err = e.MakeContext(c).Bind(&request).Errors; err != nil {
e.audit(c, service, "List", auditFailure, "可靠投递查询条件格式不正确")
e.Error(http.StatusBadRequest, err, "查询条件格式不正确")
return
}
response, err := service.List(request)
if err != nil {
e.audit(c, service, "List", auditFailure, "可靠投递列表查询失败")
e.writeError(err)
return
}
e.audit(c, service, "List", auditSuccess, "读取可靠投递列表")
e.OK(response, "查询成功")
}
func (e *API) Get(c *gin.Context) {
service, err := e.service(c)
if err != nil {
e.writeError(err)
return
}
response, err := service.Get(c.Param("id"))
if err != nil {
e.audit(c, service, "Get", auditFailure, "可靠投递详情查询失败")
e.writeError(err)
return
}
e.audit(c, service, "Get", auditSuccess, "读取可靠投递详情 "+response.Message.ID)
e.OK(response, "查询成功")
}
func (e *API) Requeue(c *gin.Context) {
service, err := e.service(c)
if err != nil {
e.writeError(err)
return
}
request := RequeueRequest{}
if err = e.MakeContext(c).Bind(&request, binding.JSON).Errors; err != nil {
e.audit(c, service, "Requeue", auditFailure, "死信重新排队请求格式不正确")
e.Error(http.StatusBadRequest, err, "请求格式不正确")
return
}
message, err := service.Requeue(c.Param("id"), request, user.GetUserId(c))
if err != nil {
e.audit(c, service, "Requeue", auditFailure, "死信重新排队被拒绝")
e.writeError(err)
return
}
e.audit(c, service, "Requeue", auditSuccess, "死信已重新排队 "+message.ID)
e.OK(message, "已重新排队")
}
func (e *API) audit(c *gin.Context, service *Service, action, status, remark string) {
if err := WriteAudit(service.Orm, Audit{Action: action, Method: c.Request.Method, Status: status, Username: user.GetUserName(c), UserID: user.GetUserId(c), ClientIP: common.GetClientIP(c), Route: c.FullPath(), Remark: remark, At: time.Now()}); err != nil {
api.GetRequestLogger(c).Errorf("outbox audit failed: %s", err.Error())
}
}
func (e *API) writeError(err error) {
switch {
case errors.Is(err, ErrInvalidInput):
e.Error(http.StatusBadRequest, err, err.Error())
case errors.Is(err, ErrNotFound):
e.Error(http.StatusNotFound, err, err.Error())
case errors.Is(err, ErrNotDead), errors.Is(err, ErrVersionConflict):
e.Error(http.StatusConflict, err, err.Error())
default:
e.Error(http.StatusInternalServerError, err, "可靠投递操作失败")
}
}
+23
View File
@@ -0,0 +1,23 @@
package outbox
import (
"time"
"gorm.io/gorm"
adminModels "git.ilapage.cn/ila/yovision/Sense/server/app/admin/models"
)
const auditSuccess, auditFailure = "1", "2"
type Audit struct {
Action, Method, Status, Username, ClientIP, Route, Remark string
UserID int
At time.Time
}
func WriteAudit(db *gorm.DB, input Audit) error {
model := adminModels.SysOperaLog{Title: "可靠投递", BusinessType: "other", Method: "outbox.API." + input.Action, RequestMethod: input.Method, OperatorType: "1", OperName: input.Username, OperUrl: input.Route, OperIp: input.ClientIP, Status: input.Status, OperTime: input.At.UTC(), Remark: input.Remark, CreatedAt: input.At.UTC(), UpdatedAt: input.At.UTC()}
model.CreateBy, model.UpdateBy = input.UserID, input.UserID
return db.Create(&model).Error
}
+41
View File
@@ -0,0 +1,41 @@
package outbox
import commonDTO "git.ilapage.cn/ila/yovision/Sense/server/common/dto"
type PageRequest struct {
commonDTO.Pagination `search:"-"`
State string `form:"state"`
InternalType string `form:"internalType"`
Keyword string `form:"keyword"`
}
type Summary struct {
Pending int64 `json:"pending"`
Retry int64 `json:"retry"`
Processing int64 `json:"processing"`
Dead int64 `json:"dead"`
}
type PageResponse struct {
List []Message `json:"list"`
Count int64 `json:"count"`
Summary Summary `json:"summary"`
}
type DetailResponse struct {
Message Message `json:"message"`
Attempts []Attempt `json:"attempts"`
}
type RequeueRequest struct {
ExpectedVersion int64 `json:"expectedVersion" binding:"required,min=1"`
Reason string `json:"reason" binding:"required"`
}
type EnqueueInput struct {
InternalType string
BusinessRef string
IdempotencyKey string
PayloadJSON []byte
MaxAttempts int
}
+58
View File
@@ -0,0 +1,58 @@
package outbox
import "time"
const (
StatePending = "pending"
StateProcessing = "processing"
StateRetry = "retry"
StateDead = "dead"
StateDelivered = "delivered"
)
// Message is a Sense-internal delivery record. PayloadJSON is deliberately
// excluded from management APIs and is not a cross-product contract.
type Message struct {
ID string `gorm:"size:36;primaryKey" json:"id"`
InternalType string `gorm:"size:64;not null;index" json:"internalType"`
BusinessRef string `gorm:"size:128;not null;index" json:"businessRef"`
IdempotencyKey string `gorm:"size:191;not null;uniqueIndex" json:"idempotencyKey"`
PayloadJSON string `gorm:"column:payload;type:jsonb;not null" json:"-"`
State string `gorm:"size:24;not null;index" json:"state"`
AttemptCount int `gorm:"not null;default:0" json:"attemptCount"`
MaxAttempts int `gorm:"not null;default:12" json:"maxAttempts"`
AvailableAt time.Time `gorm:"not null;index" json:"availableAt"`
LeaseOwner string `gorm:"size:128;not null;default:''" json:"leaseOwner,omitempty"`
LeaseUntil *time.Time `gorm:"index" json:"leaseUntil,omitempty"`
LastError string `gorm:"size:512;not null;default:''" json:"lastError,omitempty"`
DeliveredAt *time.Time `json:"deliveredAt,omitempty"`
Version int64 `gorm:"not null;default:1" json:"version"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
func (Message) TableName() string { return "sense_outbox_messages" }
// DeliveryRecord is permanent idempotency evidence. It is never deleted by
// queue cleanup and prevents a delivered key from being processed again.
type DeliveryRecord struct {
IdempotencyKey string `gorm:"size:191;primaryKey" json:"idempotencyKey"`
MessageID string `gorm:"size:36;not null;uniqueIndex" json:"messageId"`
DeliveredAt time.Time `gorm:"not null" json:"deliveredAt"`
CreatedAt time.Time `json:"createdAt"`
}
func (DeliveryRecord) TableName() string { return "sense_outbox_deliveries" }
type Attempt struct {
ID uint `gorm:"primaryKey;autoIncrement" json:"id"`
MessageID string `gorm:"size:36;not null;index" json:"messageId"`
Number int `gorm:"not null" json:"number"`
Outcome string `gorm:"size:32;not null" json:"outcome"`
Detail string `gorm:"size:512;not null;default:''" json:"detail"`
Worker string `gorm:"size:128;not null;default:''" json:"worker,omitempty"`
ActorUserID int `gorm:"not null;default:0" json:"actorUserId,omitempty"`
CreatedAt time.Time `json:"createdAt"`
}
func (Attempt) TableName() string { return "sense_outbox_attempts" }
@@ -0,0 +1,92 @@
package outbox
import (
"fmt"
"net/url"
"os"
"strings"
"sync"
"testing"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
func TestPostgresConcurrentWorkersDoNotClaimSameMessage(t *testing.T) {
dsn := os.Getenv("SENSE_OUTBOX_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("set SENSE_OUTBOX_TEST_DATABASE_URL to run the PostgreSQL multi-worker test")
}
base, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
schema := fmt.Sprintf("sense_outbox_78_%d", time.Now().UnixNano())
if err = base.Exec("CREATE SCHEMA " + schema).Error; err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = base.Exec("DROP SCHEMA IF EXISTS " + schema + " CASCADE").Error })
scoped, err := gorm.Open(postgres.Open(withSearchPath(dsn, schema)), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err = scoped.AutoMigrate(&Message{}, &DeliveryRecord{}, &Attempt{}); err != nil {
t.Fatal(err)
}
now := time.Date(2026, 8, 28, 12, 0, 0, 0, time.UTC)
for index := 0; index < 20; index++ {
if _, err = Enqueue(scoped, EnqueueInput{InternalType: "local_event_candidate", BusinessRef: fmt.Sprintf("event-%d", index), IdempotencyKey: fmt.Sprintf("event:%d:v1", index), PayloadJSON: []byte(`{}`)}, now); err != nil {
t.Fatal(err)
}
}
workers := []string{"worker-a", "worker-b"}
results := make(chan []Message, len(workers))
errorsCh := make(chan error, len(workers))
var group sync.WaitGroup
for _, worker := range workers {
group.Add(1)
go func(name string) {
defer group.Done()
relay := NewRelay(scoped)
relay.Now = func() time.Time { return now }
items, claimErr := relay.Claim(name, 20)
if claimErr != nil {
errorsCh <- claimErr
return
}
results <- items
}(worker)
}
group.Wait()
close(results)
close(errorsCh)
for claimErr := range errorsCh {
t.Fatal(claimErr)
}
seen := map[string]string{}
for batch := range results {
for _, item := range batch {
if owner, exists := seen[item.ID]; exists {
t.Fatalf("message %s claimed by %s and %s", item.ID, owner, item.LeaseOwner)
}
seen[item.ID] = item.LeaseOwner
}
}
if len(seen) != 20 {
t.Fatalf("claimed=%d want=20", len(seen))
}
}
func withSearchPath(dsn, schema string) string {
if strings.Contains(dsn, "://") {
parsed, err := url.Parse(dsn)
if err == nil {
query := parsed.Query()
query.Set("search_path", schema)
parsed.RawQuery = query.Encode()
return parsed.String()
}
}
return strings.TrimSpace(dsn) + " search_path=" + schema
}
+177
View File
@@ -0,0 +1,177 @@
package outbox
import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
var (
ErrDuplicateIdempotency = errors.New("可靠投递幂等键已存在")
ErrLeaseLost = errors.New("可靠投递租约已失效")
)
func Enqueue(tx *gorm.DB, input EnqueueInput, now time.Time) (Message, error) {
input.InternalType, input.BusinessRef, input.IdempotencyKey = strings.TrimSpace(input.InternalType), strings.TrimSpace(input.BusinessRef), strings.TrimSpace(input.IdempotencyKey)
if input.InternalType == "" || len(input.InternalType) > 64 || input.BusinessRef == "" || len(input.BusinessRef) > 128 || input.IdempotencyKey == "" || len(input.IdempotencyKey) > 191 || !json.Valid(input.PayloadJSON) {
return Message{}, ErrInvalidInput
}
if input.MaxAttempts == 0 {
input.MaxAttempts = 12
}
if input.MaxAttempts < 1 || input.MaxAttempts > 100 {
return Message{}, ErrInvalidInput
}
now = now.UTC()
message := Message{ID: uuid.NewString(), InternalType: input.InternalType, BusinessRef: input.BusinessRef, IdempotencyKey: input.IdempotencyKey, PayloadJSON: string(input.PayloadJSON), State: StatePending, MaxAttempts: input.MaxAttempts, AvailableAt: now, Version: 1, CreatedAt: now, UpdatedAt: now}
var existing int64
if err := tx.Model(&Message{}).Where("idempotency_key = ?", input.IdempotencyKey).Count(&existing).Error; err != nil {
return Message{}, fmt.Errorf("check outbox idempotency: %w", err)
}
if existing > 0 {
return Message{}, ErrDuplicateIdempotency
}
if err := tx.Create(&message).Error; err != nil {
return Message{}, fmt.Errorf("enqueue outbox message: %w", err)
}
return message, nil
}
type Relay struct {
DB *gorm.DB
Now func() time.Time
LeaseDuration time.Duration
Backoff func(int) time.Duration
}
func NewRelay(db *gorm.DB) *Relay {
return &Relay{DB: db, Now: time.Now, LeaseDuration: 30 * time.Second, Backoff: defaultBackoff}
}
func (r *Relay) Claim(worker string, limit int) ([]Message, error) {
worker = strings.TrimSpace(worker)
if worker == "" || len(worker) > 128 || limit < 1 || limit > 100 {
return nil, ErrInvalidInput
}
now, leaseUntil := r.now(), r.now().Add(r.leaseDuration())
claimed := make([]Message, 0, limit)
err := r.DB.Transaction(func(tx *gorm.DB) error {
var candidates []Message
query := tx.Where("((state IN ?) AND available_at <= ?) OR (state = ? AND lease_until < ?)", []string{StatePending, StateRetry}, now, StateProcessing, now).Order("available_at ASC, created_at ASC").Limit(limit)
if tx.Dialector.Name() == "postgres" {
query = query.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"})
}
if err := query.Find(&candidates).Error; err != nil {
return err
}
for _, item := range candidates {
result := tx.Model(&Message{}).Where("id = ? AND version = ?", item.ID, item.Version).Updates(map[string]interface{}{"state": StateProcessing, "lease_owner": worker, "lease_until": leaseUntil, "version": gorm.Expr("version + 1"), "updated_at": now})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 1 {
item.State, item.LeaseOwner, item.LeaseUntil, item.Version = StateProcessing, worker, &leaseUntil, item.Version+1
claimed = append(claimed, item)
}
}
return nil
})
if err != nil {
return nil, fmt.Errorf("claim outbox messages: %w", err)
}
return claimed, nil
}
func (r *Relay) MarkSuccess(id, worker string) error {
now := r.now()
return r.DB.Transaction(func(tx *gorm.DB) error {
var message Message
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&message, "id = ?", id).Error; err != nil {
return err
}
var delivered int64
if err := tx.Model(&DeliveryRecord{}).Where("idempotency_key = ?", message.IdempotencyKey).Count(&delivered).Error; err != nil {
return err
}
if delivered > 0 {
return nil
}
if message.State != StateProcessing || message.LeaseOwner != worker || message.LeaseUntil == nil || !message.LeaseUntil.After(now) {
return ErrLeaseLost
}
if err := tx.Create(&DeliveryRecord{IdempotencyKey: message.IdempotencyKey, MessageID: message.ID, DeliveredAt: now, CreatedAt: now}).Error; err != nil {
return err
}
number := message.AttemptCount + 1
if err := tx.Create(&Attempt{MessageID: message.ID, Number: number, Outcome: StateDelivered, Detail: "投递成功", Worker: worker, CreatedAt: now}).Error; err != nil {
return err
}
return tx.Model(&message).Updates(map[string]interface{}{"state": StateDelivered, "attempt_count": number, "delivered_at": now, "lease_owner": "", "lease_until": nil, "last_error": "", "version": gorm.Expr("version + 1"), "updated_at": now}).Error
})
}
func (r *Relay) MarkFailure(id, worker, detail string) error {
now := r.now()
detail = strings.TrimSpace(detail)
if len(detail) > 512 {
detail = detail[:512]
}
if detail == "" {
detail = "投递失败"
}
return r.DB.Transaction(func(tx *gorm.DB) error {
var message Message
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&message, "id = ?", id).Error; err != nil {
return err
}
if message.State != StateProcessing || message.LeaseOwner != worker || message.LeaseUntil == nil || !message.LeaseUntil.After(now) {
return ErrLeaseLost
}
nextAttempt := message.AttemptCount + 1
state := StateRetry
available := now.Add(r.backoff(nextAttempt))
if nextAttempt >= message.MaxAttempts {
state = StateDead
available = now
}
if err := tx.Model(&message).Updates(map[string]interface{}{"state": state, "attempt_count": nextAttempt, "available_at": available, "lease_owner": "", "lease_until": nil, "last_error": detail, "version": gorm.Expr("version + 1"), "updated_at": now}).Error; err != nil {
return err
}
return tx.Create(&Attempt{MessageID: message.ID, Number: nextAttempt, Outcome: state, Detail: detail, Worker: worker, CreatedAt: now}).Error
})
}
func (r *Relay) now() time.Time {
if r.Now != nil {
return r.Now().UTC()
}
return time.Now().UTC()
}
func (r *Relay) leaseDuration() time.Duration {
if r.LeaseDuration <= 0 {
return 30 * time.Second
}
return r.LeaseDuration
}
func (r *Relay) backoff(attempt int) time.Duration {
if r.Backoff != nil {
return r.Backoff(attempt)
}
return defaultBackoff(attempt)
}
func defaultBackoff(attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
delay := time.Second * time.Duration(1<<min(attempt-1, 8))
if delay > 5*time.Minute {
return 5 * time.Minute
}
return delay
}
+122
View File
@@ -0,0 +1,122 @@
package outbox
import (
"errors"
"testing"
"time"
"github.com/google/uuid"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func testDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open("file:"+uuid.NewString()+"?mode=memory&cache=shared"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
sqlDB, _ := db.DB()
sqlDB.SetMaxOpenConns(1)
if err = db.AutoMigrate(&Message{}, &DeliveryRecord{}, &Attempt{}); err != nil {
t.Fatal(err)
}
return db
}
func TestEnqueueClaimLeaseRecoveryBackoffAndIdempotentSuccess(t *testing.T) {
db := testDB(t)
now := time.Date(2026, 8, 28, 8, 0, 0, 0, time.UTC)
message, err := Enqueue(db, EnqueueInput{InternalType: "local_event_candidate", BusinessRef: "event-1", IdempotencyKey: "local-event:event-1:v1", PayloadJSON: []byte(`{"event":"event-1"}`), MaxAttempts: 3}, now)
if err != nil {
t.Fatal(err)
}
if _, err = Enqueue(db, EnqueueInput{InternalType: "local_event_candidate", BusinessRef: "event-1", IdempotencyKey: message.IdempotencyKey, PayloadJSON: []byte(`{}`)}, now); !errors.Is(err, ErrDuplicateIdempotency) {
t.Fatalf("expected duplicate error, got %v", err)
}
relay := NewRelay(db)
relay.Now = func() time.Time { return now }
relay.LeaseDuration = 10 * time.Second
relay.Backoff = func(int) time.Duration { return 5 * time.Second }
first, err := relay.Claim("worker-1", 1)
if err != nil || len(first) != 1 {
t.Fatalf("first claim=%#v err=%v", first, err)
}
second, err := relay.Claim("worker-2", 1)
if err != nil || len(second) != 0 {
t.Fatalf("concurrent claim=%#v err=%v", second, err)
}
now = now.Add(11 * time.Second)
recovered, err := relay.Claim("worker-2", 1)
if err != nil || len(recovered) != 1 || recovered[0].LeaseOwner != "worker-2" {
t.Fatalf("recovered=%#v err=%v", recovered, err)
}
if err = relay.MarkFailure(message.ID, "worker-2", "temporary outage"); err != nil {
t.Fatal(err)
}
now = now.Add(4 * time.Second)
waiting, _ := relay.Claim("worker-3", 1)
if len(waiting) != 0 {
t.Fatalf("claimed before backoff elapsed: %#v", waiting)
}
now = now.Add(2 * time.Second)
retry, err := relay.Claim("worker-3", 1)
if err != nil || len(retry) != 1 {
t.Fatalf("retry=%#v err=%v", retry, err)
}
if err = relay.MarkSuccess(message.ID, "worker-3"); err != nil {
t.Fatal(err)
}
if err = relay.MarkSuccess(message.ID, "worker-3"); err != nil {
t.Fatalf("idempotent success failed: %v", err)
}
var deliveries int64
db.Model(&DeliveryRecord{}).Where("idempotency_key = ?", message.IdempotencyKey).Count(&deliveries)
if deliveries != 1 {
t.Fatalf("deliveries=%d", deliveries)
}
}
func TestFailureBecomesDeadAndManualRequeuePreservesHistory(t *testing.T) {
db := testDB(t)
now := time.Date(2026, 8, 28, 9, 0, 0, 0, time.UTC)
message, err := Enqueue(db, EnqueueInput{InternalType: "audit_projection", BusinessRef: "audit-1", IdempotencyKey: "audit:audit-1:v1", PayloadJSON: []byte(`{}`), MaxAttempts: 1}, now)
if err != nil {
t.Fatal(err)
}
relay := NewRelay(db)
relay.Now = func() time.Time { return now }
claimed, _ := relay.Claim("worker", 1)
if len(claimed) != 1 {
t.Fatal("message not claimed")
}
if err = relay.MarkFailure(message.ID, "worker", "permanent failure"); err != nil {
t.Fatal(err)
}
service := NewService(db)
service.Now = func() time.Time { return now.Add(time.Minute) }
detail, err := service.Get(message.ID)
if err != nil || detail.Message.State != StateDead || len(detail.Attempts) != 1 {
t.Fatalf("detail=%#v err=%v", detail, err)
}
requeued, err := service.Requeue(message.ID, RequeueRequest{ExpectedVersion: detail.Message.Version, Reason: "出口故障已排除"}, 7)
if err != nil {
t.Fatal(err)
}
if requeued.State != StatePending {
t.Fatalf("state=%s", requeued.State)
}
detail, _ = service.Get(message.ID)
if len(detail.Attempts) != 2 || detail.Attempts[0].Outcome != "manual_requeue" || detail.Attempts[0].ActorUserID != 7 {
t.Fatalf("attempt history=%#v", detail.Attempts)
}
}
func TestProductionCannotCreateTestSink(t *testing.T) {
if _, err := NewTestSink("prod"); !errors.Is(err, ErrTestSinkForbidden) {
t.Fatalf("err=%v", err)
}
if _, err := NewTestSink("test"); err != nil {
t.Fatal(err)
}
}
+124
View File
@@ -0,0 +1,124 @@
package outbox
import (
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
coreService "github.com/go-admin-team/go-admin-core/sdk/service"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
var (
ErrInvalidInput = errors.New("可靠投递请求不符合要求")
ErrNotFound = errors.New("可靠投递记录不存在")
ErrNotDead = errors.New("仅死信记录可以重新排队")
ErrVersionConflict = errors.New("记录已变化,请刷新后重试")
)
type Service struct {
coreService.Service
Now func() time.Time
}
func NewService(db *gorm.DB) *Service {
return &Service{Service: coreService.Service{Orm: db}, Now: time.Now}
}
func (s *Service) List(request PageRequest) (PageResponse, error) {
request.State = strings.TrimSpace(request.State)
request.InternalType = strings.TrimSpace(request.InternalType)
request.Keyword = strings.TrimSpace(request.Keyword)
if request.GetPageSize() > 100 || utf8.RuneCountInString(request.Keyword) > 128 || len(request.InternalType) > 64 || (request.State != "" && !validState(request.State)) {
return PageResponse{}, ErrInvalidInput
}
query := s.Orm.Model(&Message{})
if request.State != "" {
query = query.Where("state = ?", request.State)
}
if request.InternalType != "" {
query = query.Where("internal_type = ?", request.InternalType)
}
if request.Keyword != "" {
pattern := "%" + strings.ToLower(request.Keyword) + "%"
query = query.Where("LOWER(id) LIKE ? OR LOWER(idempotency_key) LIKE ? OR LOWER(business_ref) LIKE ?", pattern, pattern, pattern)
}
var response PageResponse
if err := query.Count(&response.Count).Error; err != nil {
return PageResponse{}, fmt.Errorf("count outbox messages: %w", err)
}
if err := query.Order("created_at DESC, id DESC").Limit(request.GetPageSize()).Offset((request.GetPageIndex() - 1) * request.GetPageSize()).Find(&response.List).Error; err != nil {
return PageResponse{}, fmt.Errorf("list outbox messages: %w", err)
}
for state, target := range map[string]*int64{StatePending: &response.Summary.Pending, StateRetry: &response.Summary.Retry, StateProcessing: &response.Summary.Processing, StateDead: &response.Summary.Dead} {
if err := s.Orm.Model(&Message{}).Where("state = ?", state).Count(target).Error; err != nil {
return PageResponse{}, fmt.Errorf("summarize outbox: %w", err)
}
}
return response, nil
}
func (s *Service) Get(id string) (DetailResponse, error) {
id = strings.TrimSpace(id)
if id == "" || len(id) > 64 {
return DetailResponse{}, ErrInvalidInput
}
var message Message
if err := s.Orm.First(&message, "id = ?", id).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return DetailResponse{}, ErrNotFound
}
return DetailResponse{}, fmt.Errorf("get outbox message: %w", err)
}
var attempts []Attempt
if err := s.Orm.Where("message_id = ?", id).Order("created_at DESC, id DESC").Limit(50).Find(&attempts).Error; err != nil {
return DetailResponse{}, fmt.Errorf("list outbox attempts: %w", err)
}
return DetailResponse{Message: message, Attempts: attempts}, nil
}
func (s *Service) Requeue(id string, request RequeueRequest, actorUserID int) (Message, error) {
reason := strings.TrimSpace(request.Reason)
if utf8.RuneCountInString(reason) < 6 || utf8.RuneCountInString(reason) > 256 || request.ExpectedVersion < 1 {
return Message{}, ErrInvalidInput
}
now := s.now()
var result Message
err := s.Orm.Transaction(func(tx *gorm.DB) error {
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&result, "id = ?", strings.TrimSpace(id)).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return ErrNotFound
}
return err
}
if result.Version != request.ExpectedVersion {
return ErrVersionConflict
}
if result.State != StateDead {
return ErrNotDead
}
result.State, result.AvailableAt, result.LastError = StatePending, now, ""
result.LeaseOwner, result.LeaseUntil, result.Version = "", nil, result.Version+1
if err := tx.Save(&result).Error; err != nil {
return err
}
return tx.Create(&Attempt{MessageID: result.ID, Number: result.AttemptCount, Outcome: "manual_requeue", Detail: reason, ActorUserID: actorUserID, CreatedAt: now}).Error
})
if err != nil {
return Message{}, err
}
return result, nil
}
func (s *Service) now() time.Time {
if s.Now != nil {
return s.Now().UTC()
}
return time.Now().UTC()
}
func validState(value string) bool {
return value == StatePending || value == StateProcessing || value == StateRetry || value == StateDead || value == StateDelivered
}
+24
View File
@@ -0,0 +1,24 @@
package outbox
import (
"errors"
"strings"
)
var ErrTestSinkForbidden = errors.New("production 模式禁止启用测试接收器")
type Sink interface{ Deliver(Message) error }
type TestSink struct{ Delivered []string }
func (s *TestSink) Deliver(message Message) error {
s.Delivered = append(s.Delivered, message.IdempotencyKey)
return nil
}
func NewTestSink(applicationMode string) (*TestSink, error) {
if strings.EqualFold(strings.TrimSpace(applicationMode), "prod") || strings.EqualFold(strings.TrimSpace(applicationMode), "production") {
return nil, ErrTestSinkForbidden
}
return &TestSink{}, nil
}
@@ -0,0 +1,65 @@
package version
import (
"fmt"
"runtime"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/outbox"
"git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration"
migrationModels "git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration/models"
common "git.ilapage.cn/ila/yovision/Sense/server/common/models"
)
func init() {
_, fileName, _, _ := runtime.Caller(0)
migration.Migrate.SetVersion(migration.GetFilename(fileName), migrateSenseOutbox)
}
func migrateSenseOutbox(db *gorm.DB, version string) error {
return db.Transaction(func(tx *gorm.DB) error {
if err := tx.AutoMigrate(&outbox.Message{}, &outbox.DeliveryRecord{}, &outbox.Attempt{}); err != nil {
return err
}
root, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: senseLayoutMenuName, Title: "视频感知", Icon: "video-camera", Path: "/sense", MenuType: "M", Component: "Layout", Sort: 5, Visible: "0", IsFrame: "1"})
if err != nil {
return err
}
page, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOutbox", Title: "可靠投递", Icon: "connection", Path: "outbox", Paths: fmt.Sprintf("/0/%d", root.MenuId), MenuType: "C", Permission: "sense:outbox:list", ParentId: root.MenuId, Component: "/sense/outbox/index", Sort: 11, Visible: "0", IsFrame: "1"})
if err != nil {
return err
}
detail, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOutboxDetail", Title: "查看投递详情", MenuType: "F", Action: "GET", Permission: "sense:outbox:detail", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 1, Visible: "1", IsFrame: "1"})
if err != nil {
return err
}
requeue, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOutboxRequeue", Title: "死信重新排队", MenuType: "F", Action: "POST", Permission: "sense:outbox:requeue", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 2, Visible: "1", IsFrame: "1"})
if err != nil {
return err
}
for _, role := range []string{"implementation_operator", "site_admin", "viewer"} {
if err = attachDeviceRole(tx, role, []migrationModels.SysMenu{page, detail}); err != nil {
return err
}
for _, policy := range [][2]string{{"/api/v1/outbox", "GET"}, {"/api/v1/outbox/:id", "GET"}} {
if err = tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&deviceCasbinRule{Ptype: "p", V0: role, V1: policy[0], V2: policy[1]}).Error; err != nil {
return err
}
}
}
for _, role := range []string{"implementation_operator", "site_admin"} {
if err = attachDeviceRole(tx, role, []migrationModels.SysMenu{requeue}); err != nil {
return err
}
if err = tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&deviceCasbinRule{Ptype: "p", V0: role, V1: "/api/v1/outbox/:id/requeue", V2: "POST"}).Error; err != nil {
return err
}
}
if err = rebuildSenseMenuPaths(tx, root.MenuId, "/0"); err != nil {
return err
}
return tx.Create(&common.Migration{Version: version}).Error
})
}
@@ -0,0 +1,41 @@
package version
import (
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/outbox"
migrationModels "git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration/models"
common "git.ilapage.cn/ila/yovision/Sense/server/common/models"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"testing"
)
func TestOutboxMigrationAddsRBACWithoutFixtures(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err = db.AutoMigrate(&migrationModels.SysRole{}, &migrationModels.SysMenu{}, &deviceCasbinRule{}, &common.Migration{}); err != nil {
t.Fatal(err)
}
for _, role := range []string{"implementation_operator", "site_admin", "viewer"} {
if err = db.Create(&migrationModels.SysRole{RoleName: role, RoleKey: role, Status: "2"}).Error; err != nil {
t.Fatal(err)
}
}
const version = "2026082815000_outbox.go"
if err = migrateSenseOutbox(db, version); err != nil {
t.Fatal(err)
}
if !db.Migrator().HasTable(&outbox.Message{}) || !db.Migrator().HasTable(&outbox.DeliveryRecord{}) || !db.Migrator().HasTable(&outbox.Attempt{}) {
t.Fatal("outbox tables missing")
}
var menus, reads, writes, fixtures, applied int64
db.Model(&migrationModels.SysMenu{}).Where("menu_name LIKE ?", "SenseOutbox%").Count(&menus)
db.Model(&deviceCasbinRule{}).Where("v1 LIKE ? AND v2 = ?", "/api/v1/outbox%", "GET").Count(&reads)
db.Model(&deviceCasbinRule{}).Where("v1 = ? AND v2 = ?", "/api/v1/outbox/:id/requeue", "POST").Count(&writes)
db.Model(&outbox.Message{}).Count(&fixtures)
db.Model(&common.Migration{}).Where("version = ?", version).Count(&applied)
if menus != 3 || reads != 6 || writes != 2 || fixtures != 0 || applied != 1 {
t.Fatalf("menus=%d reads=%d writes=%d fixtures=%d applied=%d", menus, reads, writes, fixtures, applied)
}
}
@@ -0,0 +1,74 @@
package version
import (
"fmt"
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/ops_alert"
"git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration"
migrationModels "git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration/models"
common "git.ilapage.cn/ila/yovision/Sense/server/common/models"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"runtime"
)
func init() {
_, fileName, _, _ := runtime.Caller(0)
migration.Migrate.SetVersion(migration.GetFilename(fileName), migrateSenseOpsAlert)
}
func migrateSenseOpsAlert(db *gorm.DB, version string) error {
return db.Transaction(func(tx *gorm.DB) error {
if err := tx.AutoMigrate(&ops_alert.Alert{}, &ops_alert.Transition{}); err != nil {
return err
}
root, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: senseLayoutMenuName, Title: "视频感知", Icon: "video-camera", Path: "/sense", MenuType: "M", Component: "Layout", Sort: 5, Visible: "0", IsFrame: "1"})
if err != nil {
return err
}
page, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlert", Title: "运维告警", Icon: "warning", Path: "ops-alert", Paths: fmt.Sprintf("/0/%d", root.MenuId), MenuType: "C", Permission: "sense:ops-alert:list", ParentId: root.MenuId, Component: "/sense/ops-alert/index", Sort: 11, Visible: "0", IsFrame: "1"})
if err != nil {
return err
}
detail, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlertDetail", Title: "查看告警详情", MenuType: "F", Action: "GET", Permission: "sense:ops-alert:detail", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 1, Visible: "1", IsFrame: "1"})
if err != nil {
return err
}
refresh, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlertEvaluate", Title: "刷新告警状态", MenuType: "F", Action: "POST", Permission: "sense:ops-alert:evaluate", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 2, Visible: "1", IsFrame: "1"})
if err != nil {
return err
}
ack, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlertAcknowledge", Title: "确认运维告警", MenuType: "F", Action: "POST", Permission: "sense:ops-alert:acknowledge", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 3, Visible: "1", IsFrame: "1"})
if err != nil {
return err
}
recover, err := ensureDeviceMenu(tx, migrationModels.SysMenu{MenuName: "SenseOpsAlertRecover", Title: "确认告警恢复", MenuType: "F", Action: "POST", Permission: "sense:ops-alert:recover", ParentId: page.MenuId, Paths: fmt.Sprintf("/0/%d/%d", root.MenuId, page.MenuId), Sort: 4, Visible: "1", IsFrame: "1"})
if err != nil {
return err
}
readPolicies := [][2]string{{"/api/v1/ops-alerts", "GET"}, {"/api/v1/ops-alerts/:id", "GET"}}
for _, role := range []string{"implementation_operator", "site_admin", "viewer"} {
if err = attachDeviceRole(tx, role, []migrationModels.SysMenu{page, detail}); err != nil {
return err
}
for _, policy := range readPolicies {
if err = tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&deviceCasbinRule{Ptype: "p", V0: role, V1: policy[0], V2: policy[1]}).Error; err != nil {
return err
}
}
}
for _, role := range []string{"implementation_operator", "site_admin"} {
if err = attachDeviceRole(tx, role, []migrationModels.SysMenu{refresh, ack, recover}); err != nil {
return err
}
for _, policy := range [][2]string{{"/api/v1/ops-alerts/evaluate", "POST"}, {"/api/v1/ops-alerts/:id/acknowledge", "POST"}, {"/api/v1/ops-alerts/:id/recover", "POST"}} {
if err = tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&deviceCasbinRule{Ptype: "p", V0: role, V1: policy[0], V2: policy[1]}).Error; err != nil {
return err
}
}
}
if err = rebuildSenseMenuPaths(tx, root.MenuId, "/0"); err != nil {
return err
}
return tx.Create(&common.Migration{Version: version}).Error
})
}
@@ -0,0 +1,39 @@
package version
import (
"git.ilapage.cn/ila/yovision/Sense/server/app/sense/ops_alert"
migrationModels "git.ilapage.cn/ila/yovision/Sense/server/cmd/migrate/migration/models"
common "git.ilapage.cn/ila/yovision/Sense/server/common/models"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"testing"
)
func TestSenseOpsAlertMigrationRBAC(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err = db.AutoMigrate(&migrationModels.SysRole{}, &migrationModels.SysMenu{}, &deviceCasbinRule{}, &common.Migration{}); err != nil {
t.Fatal(err)
}
for _, role := range []string{"implementation_operator", "site_admin", "viewer"} {
if err = db.Create(&migrationModels.SysRole{RoleName: role, RoleKey: role, Status: "2"}).Error; err != nil {
t.Fatal(err)
}
}
const version = "2026082816000_ops_alert.go"
if err = migrateSenseOpsAlert(db, version); err != nil {
t.Fatal(err)
}
var pages, viewerWrites, operatorWrites int64
db.Model(&migrationModels.SysMenu{}).Where("permission = ?", "sense:ops-alert:list").Count(&pages)
db.Model(&deviceCasbinRule{}).Where("v0 = ? AND v1 LIKE ? AND v2 = ?", "viewer", "/api/v1/ops-alerts%", "POST").Count(&viewerWrites)
db.Model(&deviceCasbinRule{}).Where("v0 = ? AND v1 LIKE ? AND v2 = ?", "implementation_operator", "/api/v1/ops-alerts%", "POST").Count(&operatorWrites)
if pages != 1 || viewerWrites != 0 || operatorWrites != 3 {
t.Fatalf("unexpected RBAC page=%d viewerWrites=%d operatorWrites=%d", pages, viewerWrites, operatorWrites)
}
if !db.Migrator().HasTable(&ops_alert.Alert{}) || !db.Migrator().HasTable(&ops_alert.Transition{}) {
t.Fatal("ops alert tables missing")
}
}
+21
View File
@@ -0,0 +1,21 @@
import request from '@/utils/request'
export function listOpsAlerts(query) {
return request({ url: '/api/v1/ops-alerts', method: 'get', params: query })
}
export function getOpsAlert(id) {
return request({ url: `/api/v1/ops-alerts/${encodeURIComponent(id)}`, method: 'get' })
}
export function evaluateOpsAlerts() {
return request({ url: '/api/v1/ops-alerts/evaluate', method: 'post' })
}
export function acknowledgeOpsAlert(id, data) {
return request({ url: `/api/v1/ops-alerts/${encodeURIComponent(id)}/acknowledge`, method: 'post', data })
}
export function recoverOpsAlert(id, data) {
return request({ url: `/api/v1/ops-alerts/${encodeURIComponent(id)}/recover`, method: 'post', data })
}
+20
View File
@@ -0,0 +1,20 @@
import request from '@/utils/request'
export function listOutbox(query) {
return request({ url: '/api/v1/outbox', method: 'get', params: query })
}
export function getOutbox(id) {
return request({
url: `/api/v1/outbox/${encodeURIComponent(id)}`,
method: 'get'
})
}
export function requeueOutbox(id, expectedVersion, reason) {
return request({
url: `/api/v1/outbox/${encodeURIComponent(id)}/requeue`,
method: 'post',
data: { expectedVersion, reason }
})
}
@@ -0,0 +1,109 @@
<template>
<BasicLayout>
<template #wrapper>
<el-card class="box-card">
<div class="page-header">
<div>
<h3>运维告警</h3>
<p>处理设备、媒体服务和边缘节点的运行异常。</p>
</div>
<el-button v-permisaction="['sense:ops-alert:evaluate']" :icon="Refresh" :loading="refreshing" @click="refreshSources">刷新状态</el-button>
</div>
<el-alert class="boundary-alert" type="info" :closable="false" show-icon title="这里只处理系统运行问题">
运维告警不会创建本地安全事件,也不会发送到 Bell 作为业务预警;确认告警仅表示已有人处理,不代表故障已经恢复。
</el-alert>
<el-row :gutter="12" class="summary-row" aria-label="告警概览">
<el-col :xs="24" :sm="8"><div class="summary-item"><span>活动告警</span><strong>{{ summary.active }}</strong></div></el-col>
<el-col :xs="24" :sm="8"><div class="summary-item"><span>待确认</span><strong class="danger-number">{{ summary.unacknowledged }}</strong></div></el-col>
<el-col :xs="24" :sm="8"><div class="summary-item"><span>恢复观察</span><strong class="primary-number">{{ summary.recovering }}</strong></div></el-col>
</el-row>
<el-form :model="query" label-width="76px" class="filter-form" @submit.prevent="search">
<el-form-item label="告警类型">
<el-select v-model="query.alertType" clearable placeholder="全部类型">
<el-option v-for="item in typeOptions" :key="item.value" :label="item.label" :value="item.value" />
</el-select>
</el-form-item>
<el-form-item label="处理状态">
<el-select v-model="query.state" clearable placeholder="全部状态">
<el-option label="待确认" value="unacknowledged" /><el-option label="已确认" value="acknowledged" />
<el-option label="恢复观察" value="recovering" /><el-option label="已恢复" value="recovered" />
</el-select>
</el-form-item>
<el-form-item label="关键词"><el-input v-model="query.keyword" clearable placeholder="设备、位置或详情" @keyup.enter="search" /></el-form-item>
<el-form-item class="filter-actions"><el-button type="primary" :icon="Search" native-type="submit">查询</el-button><el-button :icon="RefreshLeft" @click="reset">重置</el-button></el-form-item>
</el-form>
<el-table v-loading="loading" :data="alerts" border stripe empty-text="当前筛选条件下没有运维告警">
<el-table-column label="告警" min-width="180"><template #default="scope"><strong>{{ scope.row.title }}</strong><div class="muted">{{ alertTypeLabel(scope.row.alertType) }}</div></template></el-table-column>
<el-table-column label="对象" min-width="170"><template #default="scope"><strong>{{ scope.row.objectName }}</strong><div class="muted">{{ scope.row.location || scope.row.objectId }}</div></template></el-table-column>
<el-table-column label="状态" width="110" align="center"><template #default="scope"><el-tag :type="alertStateType(scope.row.state)" size="small">{{ alertStateLabel(scope.row.state) }}</el-tag></template></el-table-column>
<el-table-column label="首次 / 最近发现" min-width="190"><template #default="scope"><div>{{ formatTime(scope.row.firstSeenAt) }}</div><small>{{ formatTime(scope.row.lastSeenAt) }}</small></template></el-table-column>
<el-table-column prop="occurrenceCount" label="重复次数" width="92" align="center" />
<el-table-column label="操作" width="184" fixed="right">
<template #default="scope">
<el-button v-permisaction="['sense:ops-alert:detail']" type="primary" link @click="openDetail(scope.row.id)">详情</el-button>
<el-button v-if="canAcknowledge(scope.row)" v-permisaction="['sense:ops-alert:acknowledge']" type="warning" link @click="openAction(scope.row, 'acknowledge')">确认</el-button>
<el-button v-if="canRecover(scope.row)" v-permisaction="['sense:ops-alert:recover']" type="success" link @click="openAction(scope.row, 'recover')">恢复</el-button>
</template>
</el-table-column>
</el-table>
<pagination v-show="total > 0" v-model:current-page="query.pageIndex" v-model:page-size="query.pageSize" :total="total" @pagination="load" />
<p class="help-text">同一对象的同类故障使用唯一指纹去重;恢复后再次发生会复用历史记录并开启新的处理周期。</p>
</el-card>
<el-dialog v-model="detailOpen" title="运维告警详情" width="min(780px, calc(100vw - 24px))">
<el-descriptions v-if="selected" :column="2" border>
<el-descriptions-item label="告警编号">{{ selected.id }}</el-descriptions-item><el-descriptions-item label="处理状态"><el-tag :type="alertStateType(selected.state)">{{ alertStateLabel(selected.state) }}</el-tag></el-descriptions-item>
<el-descriptions-item label="异常对象">{{ selected.objectName }}</el-descriptions-item><el-descriptions-item label="发现次数">{{ selected.occurrenceCount }}</el-descriptions-item>
<el-descriptions-item label="异常详情" :span="2">{{ selected.detail }}</el-descriptions-item><el-descriptions-item label="建议处理" :span="2">{{ selected.nextAction }}</el-descriptions-item>
</el-descriptions>
<el-alert class="detail-boundary" type="warning" :closable="false" show-icon title="业务隔离边界">该记录仅用于 Sense 运维,不会生成安全事件或 Bell 业务预警。</el-alert>
<el-timeline v-if="transitions.length" class="timeline"><el-timeline-item v-for="item in transitions" :key="item.id" :timestamp="formatTime(item.createdAt)"><strong>{{ actionLabel(item.action) }}</strong><div class="muted">{{ item.reason }}</div></el-timeline-item></el-timeline>
<template #footer><el-button @click="detailOpen = false">关闭</el-button></template>
</el-dialog>
<el-dialog v-model="actionOpen" :title="actionMode === 'acknowledge' ? '确认运维告警' : '确认故障恢复'" width="min(520px, calc(100vw - 24px))" :close-on-click-modal="false">
<el-alert :type="actionMode === 'acknowledge' ? 'warning' : 'success'" :closable="false" show-icon :title="actionMode === 'acknowledge' ? '确认不代表恢复' : '仅在健康状态已稳定后关闭告警'" />
<el-form ref="actionFormRef" :model="actionForm" :rules="actionRules" label-position="top" class="action-form">
<el-form-item label="处理说明" prop="reason"><el-input v-model="actionForm.reason" type="textarea" :rows="4" maxlength="256" show-word-limit placeholder="请输入至少 6 个字符,说明处理人、措施或恢复依据" /></el-form-item>
</el-form>
<template #footer><el-button @click="actionOpen = false">取消</el-button><el-button :type="actionMode === 'acknowledge' ? 'warning' : 'success'" :loading="submitting" @click="submitAction">确认提交</el-button></template>
</el-dialog>
</template>
</BasicLayout>
</template>
<script setup>
import { onMounted, reactive, ref } from 'vue'
import { Refresh, RefreshLeft, Search } from '@element-plus/icons-vue'
import { ElMessage } from 'element-plus'
import { acknowledgeOpsAlert, evaluateOpsAlerts, getOpsAlert, listOpsAlerts, recoverOpsAlert } from '@/api/sense/ops-alert'
import { alertStateLabel, alertStateType, alertTypeLabel, buildOpsAlertQuery, canAcknowledge, canRecover } from './opsAlertState'
defineOptions({ name: 'SenseOpsAlert' })
const loading = ref(false); const refreshing = ref(false); const submitting = ref(false); const detailOpen = ref(false); const actionOpen = ref(false)
const alerts = ref([]); const total = ref(0); const selected = ref(null); const transitions = ref([]); const actionMode = ref('acknowledge'); const actionFormRef = ref()
const summary = reactive({ active: 0, unacknowledged: 0, recovering: 0 })
const query = reactive({ pageIndex: 1, pageSize: 10, alertType: '', state: '', keyword: '' })
const actionForm = reactive({ reason: '', expectedVersion: 0 })
const actionRules = { reason: [{ required: true, message: '请输入处理说明', trigger: 'blur' }, { min: 6, max: 256, message: '处理说明需为 6 至 256 个字符', trigger: 'blur' }] }
const typeOptions = ['device_offline', 'authentication_failed', 'clock_drift', 'reconciliation_failed', 'media_shard_failed', 'control_tunnel_failed'].map(value => ({ value, label: alertTypeLabel(value) }))
function unwrap(response) { return response?.data?.data ?? response?.data ?? response }
function formatTime(value) { return value ? new Date(value).toLocaleString('zh-CN', { hour12: false }) : '—' }
function actionLabel(value) { return ({ detected: '发现异常', acknowledge: '人工确认', health_restored: '健康恢复', recover: '确认恢复', relapsed: '恢复失败', reopened: '再次发生' })[value] || value }
async function load() { loading.value = true; try { const payload = unwrap(await listOpsAlerts(buildOpsAlertQuery(query))) || {}; alerts.value = payload.list || []; total.value = payload.count || 0; Object.assign(summary, payload.summary || { active: 0, unacknowledged: 0, recovering: 0 }) } catch (error) { ElMessage.error(error.message || '运维告警加载失败') } finally { loading.value = false } }
async function refreshSources() { refreshing.value = true; try { await evaluateOpsAlerts(); ElMessage.success('状态已刷新'); await load() } catch (error) { ElMessage.error(error.message || '健康状态刷新失败') } finally { refreshing.value = false } }
function search() { query.pageIndex = 1; load() }
function reset() { Object.assign(query, { pageIndex: 1, alertType: '', state: '', keyword: '' }); load() }
async function openDetail(id) { try { const payload = unwrap(await getOpsAlert(id)) || {}; selected.value = payload.alert; transitions.value = payload.transitions || []; detailOpen.value = true } catch (error) { ElMessage.error(error.message || '告警详情加载失败') } }
function openAction(item, mode) { selected.value = item; actionMode.value = mode; actionForm.reason = ''; actionForm.expectedVersion = item.version; actionOpen.value = true }
async function submitAction() { if (!await actionFormRef.value.validate().catch(() => false)) return; submitting.value = true; try { const payload = { expectedVersion: actionForm.expectedVersion, reason: actionForm.reason.trim() }; if (actionMode.value === 'acknowledge') await acknowledgeOpsAlert(selected.value.id, payload); else await recoverOpsAlert(selected.value.id, payload); ElMessage.success(actionMode.value === 'acknowledge' ? '告警已确认' : '告警已恢复'); actionOpen.value = false; detailOpen.value = false; await load() } catch (error) { ElMessage.warning(error.message || '状态已变化,请刷新后重试') } finally { submitting.value = false } }
onMounted(load)
</script>
<style scoped>
.page-header{display:flex;align-items:flex-start;justify-content:space-between;gap:16px}.page-header h3{margin:0 0 6px}.page-header p{margin:0;color:#909399}.boundary-alert{margin:16px 0}.summary-row{margin-bottom:20px}.summary-item{display:flex;align-items:center;justify-content:space-between;min-height:72px;padding:12px 16px;border:1px solid #ebeef5;border-radius:4px}.summary-item span{color:#606266}.summary-item strong{font-size:22px}.danger-number{color:#f56c6c}.primary-number{color:#409eff}.filter-form{display:flex;align-items:flex-end;flex-wrap:wrap;gap:0 12px;margin:16px 0 2px}.filter-form .el-form-item{width:230px}.filter-form .filter-actions{width:auto}.filter-form :deep(.el-select){width:100%}.muted,small,.help-text{color:#909399;font-size:12px}.help-text{margin:12px 0 0}.detail-boundary{margin:16px 0}.timeline{padding-top:8px}.action-form{margin-top:16px}@media(max-width:768px){.page-header{align-items:stretch;flex-direction:column}.filter-form .el-form-item{width:100%}.summary-item{margin-bottom:8px}}
</style>
@@ -0,0 +1,30 @@
const typeLabels = {
device_offline: '设备离线',
authentication_failed: '认证失败',
clock_drift: '时间漂移',
reconciliation_failed: '状态对账失败',
media_shard_failed: '媒体分片异常',
control_tunnel_failed: '控制隧道异常'
}
const stateLabels = {
unacknowledged: '待确认',
acknowledged: '已确认',
recovering: '恢复观察',
recovered: '已恢复'
}
export function alertTypeLabel(value) { return typeLabels[value] || value || '未知类型' }
export function alertStateLabel(value) { return stateLabels[value] || value || '未知状态' }
export function alertStateType(value) { return ({ unacknowledged: 'danger', acknowledged: 'warning', recovering: 'primary', recovered: 'success' })[value] || 'info' }
export function canAcknowledge(item) { return item?.state === 'unacknowledged' && item?.version > 0 }
export function canRecover(item) { return item?.state === 'recovering' && item?.version > 0 }
export function buildOpsAlertQuery(query) {
return {
pageIndex: Number(query.pageIndex) || 1,
pageSize: Number(query.pageSize) || 10,
alertType: String(query.alertType || '').trim(),
state: String(query.state || '').trim(),
keyword: String(query.keyword || '').trim()
}
}
+447
View File
@@ -0,0 +1,447 @@
<template>
<BasicLayout>
<template #wrapper>
<el-card class="box-card">
<div class="page-header">
<div>
<h3>可靠投递</h3>
<p>查看 Sense 内部待投递记录、重试进度和人工恢复。</p>
</div>
<el-button
:icon="Refresh"
:loading="loading"
@click="load"
>刷新状态</el-button>
</div>
<el-alert
type="info"
:closable="false"
show-icon
class="boundary-alert"
>
<template #title>未配置外部投递出口不影响 Sense 核心功能</template>
内部记录会安全保留,正式 connector 由后续协调工单提供;测试接收器在
production 模式不可启用。
</el-alert>
<el-row :gutter="12" class="summary-row" aria-label="可靠投递队列概览">
<el-col
v-for="card in summaryCards"
:key="card.key"
:xs="12"
:sm="6"
><div class="summary-item">
<span>{{ card.label }}</span><strong :class="card.className">{{ summary[card.key] }}</strong>
</div></el-col>
</el-row>
<el-form
:model="query"
label-width="76px"
class="filter-form"
@submit.prevent="search"
>
<el-form-item label="关键词"><el-input
v-model="query.keyword"
clearable
placeholder="内部记录、业务引用或幂等键"
@keyup.enter="search"
/></el-form-item>
<el-form-item label="状态"><el-select
v-model="query.state"
clearable
placeholder="全部状态"
><el-option
v-for="(label, value) in stateLabels"
:key="value"
:label="label"
:value="value"
/></el-select></el-form-item>
<el-form-item label="内部类型"><el-select
v-model="query.internalType"
clearable
placeholder="全部类型"
><el-option
v-for="(label, value) in typeLabels"
:key="value"
:label="label"
:value="value"
/></el-select></el-form-item>
<el-form-item class="filter-actions"><el-button
type="primary"
:icon="Search"
native-type="submit"
>查询</el-button><el-button
:icon="RefreshLeft"
@click="reset"
>重置</el-button></el-form-item>
</el-form>
<el-table
v-loading="loading"
:data="items"
border
stripe
empty-text="当前筛选条件下没有可靠投递记录"
>
<el-table-column
label="内部记录"
min-width="255"
><template #default="scope"><strong>{{ typeLabel(scope.row.internalType) }}</strong>
<div class="code-text">{{ scope.row.id }}</div>
<small>幂等键:{{ scope.row.idempotencyKey }}</small></template></el-table-column>
<el-table-column
label="状态"
width="112"
align="center"
><template #default="scope"><el-tag :type="stateType(scope.row.state)">{{
stateLabel(scope.row.state)
}}</el-tag></template></el-table-column>
<el-table-column
label="尝试"
width="74"
align="center"
><template #default="scope">{{ scope.row.attemptCount }} 次</template></el-table-column>
<el-table-column
label="下次动作"
min-width="180"
><template #default="scope">{{
nextAction(scope.row)
}}</template></el-table-column>
<el-table-column
label="租约"
min-width="142"
><template #default="scope"><div>{{ scope.row.leaseOwner || "未领取" }}</div>
<small v-if="scope.row.leaseUntil">{{
formatTime(scope.row.leaseUntil)
}}</small></template></el-table-column>
<el-table-column
label="最近结果"
min-width="210"
><template #default="scope">{{
scope.row.lastError ||
(scope.row.state === "delivered" ? "投递成功" : "尚未失败")
}}</template></el-table-column>
<el-table-column
label="操作"
width="150"
fixed="right"
><template #default="scope"><el-button
v-permisaction="['sense:outbox:detail']"
type="primary"
link
@click="openDetail(scope.row.id)"
>详情</el-button><el-button
v-if="scope.row.state === 'dead'"
v-permisaction="['sense:outbox:requeue']"
type="warning"
link
@click="openRequeue(scope.row)"
>重新排队</el-button></template></el-table-column>
</el-table>
<pagination
v-show="total > 0"
v-model:current-page="query.pageIndex"
v-model:page-size="query.pageSize"
:total="total"
@pagination="load"
/>
<p class="boundary-text">
页面不展示内部
payload、外部凭据或机器身份;人工恢复保留原业务记录、幂等键和失败历史。
</p>
</el-card>
<el-dialog
v-model="detailOpen"
title="投递记录详情"
width="min(780px, calc(100vw - 24px))"
:close-on-click-modal="false"
>
<el-descriptions v-if="selected" :column="2" border>
<el-descriptions-item label="内部记录">{{
selected.message.id
}}</el-descriptions-item><el-descriptions-item label="内部类型">{{
typeLabel(selected.message.internalType)
}}</el-descriptions-item>
<el-descriptions-item
label="幂等键"
:span="2"
><span class="code-text">{{
selected.message.idempotencyKey
}}</span></el-descriptions-item><el-descriptions-item label="当前状态"><el-tag :type="stateType(selected.message.state)">{{
stateLabel(selected.message.state)
}}</el-tag></el-descriptions-item><el-descriptions-item label="业务引用">{{
selected.message.businessRef
}}</el-descriptions-item>
<el-descriptions-item label="最近结果" :span="2">{{
selected.message.lastError || "尚未失败"
}}</el-descriptions-item>
</el-descriptions>
<h4 class="timeline-title">处理时间线</h4>
<el-timeline v-if="selected"><el-timeline-item
v-for="attempt in selected.attempts"
:key="attempt.id"
:timestamp="formatTime(attempt.createdAt)"
placement="top"
><strong>第 {{ attempt.number }} 次 · {{ attempt.outcome }}</strong>
<div>{{ attempt.detail }}</div>
<small v-if="attempt.actorUserId">操作人编号:{{ attempt.actorUserId }}</small></el-timeline-item><el-timeline-item
v-if="!selected.attempts.length"
:timestamp="formatTime(selected.message.createdAt)"
>与业务记录在同一事务中创建</el-timeline-item></el-timeline>
<template #footer><el-button @click="detailOpen = false">关闭</el-button></template>
</el-dialog>
<el-dialog
v-model="requeueOpen"
title="将死信重新排队"
width="min(560px, calc(100vw - 24px))"
:close-on-click-modal="false"
@closed="resetRequeue"
>
<el-alert
title="这是受控恢复操作"
type="warning"
:closable="false"
show-icon
>只创建新的投递尝试,不修改原业务记录或删除失败历史。执行前请先排除失败原因。</el-alert>
<el-form
ref="requeueFormRef"
:model="requeueForm"
:rules="requeueRules"
label-position="top"
class="requeue-form"
><el-form-item
label="恢复原因"
prop="reason"
><el-input
v-model="requeueForm.reason"
type="textarea"
:rows="3"
maxlength="256"
show-word-limit
placeholder="例如:出口配置已恢复,已核对幂等键和目标状态"
/></el-form-item></el-form>
<p class="boundary-text">
原因将与操作人、对象版本和时间一起写入脱敏审计。
</p>
<template #footer><el-button @click="requeueOpen = false">取消</el-button><el-button
type="warning"
:loading="requeueLoading"
@click="confirmRequeue"
>确认重新排队</el-button></template>
</el-dialog>
</template>
</BasicLayout>
</template>
<script setup>
import { computed, onMounted, reactive, ref } from 'vue'
import { Refresh, RefreshLeft, Search } from '@element-plus/icons-vue'
import { ElMessage } from 'element-plus'
import { getOutbox, listOutbox, requeueOutbox } from '@/api/sense/outbox'
import {
buildOutboxQuery,
formatTime,
nextAction,
stateLabel,
stateLabels,
stateType,
typeLabel,
typeLabels
} from './outboxState'
defineOptions({ name: 'SenseOutbox' })
const loading = ref(false)
const items = ref([])
const total = ref(0)
const detailOpen = ref(false)
const selected = ref(null)
const requeueOpen = ref(false)
const requeueLoading = ref(false)
const requeueTarget = ref(null)
const requeueFormRef = ref(null)
const summary = reactive({ pending: 0, retry: 0, processing: 0, dead: 0 })
const query = reactive({
pageIndex: 1,
pageSize: 10,
state: '',
internalType: '',
keyword: ''
})
const requeueForm = reactive({ reason: '' })
const requeueRules = {
reason: [
{ required: true, message: '请填写恢复原因', trigger: 'blur' },
{ min: 6, max: 256, message: '请填写 6 至 256 个字符', trigger: 'blur' }
]
}
const summaryCards = computed(() => [
{ key: 'pending', label: '等待投递' },
{ key: 'retry', label: '重试等待', className: 'warning-number' },
{ key: 'processing', label: '处理中 / 租约' },
{ key: 'dead', label: '死信待处理', className: 'danger-number' }
])
function unwrap(response) {
return response?.data?.data ?? response?.data ?? response
}
async function load() {
loading.value = true
try {
const payload = unwrap(await listOutbox(buildOutboxQuery(query))) || {}
items.value = payload.list || []
total.value = payload.count || 0
Object.assign(
summary,
payload.summary || { pending: 0, retry: 0, processing: 0, dead: 0 }
)
} catch (error) {
ElMessage.error(error.message || '可靠投递状态加载失败')
} finally {
loading.value = false
}
}
function search() {
query.pageIndex = 1
load()
}
function reset() {
Object.assign(query, {
pageIndex: 1,
state: '',
internalType: '',
keyword: ''
})
load()
}
async function openDetail(id) {
try {
selected.value = unwrap(await getOutbox(id))
detailOpen.value = true
} catch (error) {
ElMessage.error(error.message || '投递详情加载失败')
}
}
function openRequeue(item) {
requeueTarget.value = item
requeueOpen.value = true
}
function resetRequeue() {
requeueForm.reason = ''
requeueTarget.value = null
requeueFormRef.value?.clearValidate()
}
async function confirmRequeue() {
if (!requeueFormRef.value || !requeueTarget.value) return
const valid = await requeueFormRef.value.validate().catch(() => false)
if (!valid) return
requeueLoading.value = true
try {
await requeueOutbox(
requeueTarget.value.id,
requeueTarget.value.version,
requeueForm.reason.trim()
)
ElMessage.success('死信已重新排队,原失败历史已保留')
requeueOpen.value = false
detailOpen.value = false
await load()
} catch (error) {
ElMessage.warning(error.message || '重新排队失败,请刷新状态后重试')
} finally {
requeueLoading.value = false
}
}
onMounted(load)
</script>
<style scoped>
.page-header {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
}
.page-header h3 {
margin: 0 0 6px;
}
.page-header p {
margin: 0;
color: #909399;
}
.boundary-alert {
margin: 16px 0;
}
.summary-row {
margin-bottom: 18px;
}
.summary-item {
display: flex;
align-items: center;
justify-content: space-between;
min-height: 68px;
padding: 12px 16px;
border: 1px solid #ebeef5;
border-radius: 4px;
}
.summary-item span {
color: #606266;
}
.summary-item strong {
font-size: 22px;
}
.warning-number {
color: #e6a23c;
}
.danger-number {
color: #f56c6c;
}
.filter-form {
display: flex;
align-items: flex-end;
flex-wrap: wrap;
gap: 0 12px;
margin-bottom: 2px;
}
.filter-form .el-form-item {
width: 260px;
}
.filter-form .filter-actions {
width: auto;
}
.filter-form :deep(.el-select) {
width: 100%;
}
.code-text {
font-family: ui-monospace, SFMono-Regular, Consolas, monospace;
overflow-wrap: anywhere;
}
.boundary-text,
small {
color: #909399;
font-size: 12px;
}
.boundary-text {
margin: 12px 0 0;
}
.timeline-title {
margin: 20px 0 14px;
}
.requeue-form {
margin-top: 16px;
}
@media (max-width: 768px) {
.page-header {
align-items: stretch;
flex-direction: column;
}
.filter-form .el-form-item {
width: 100%;
}
.summary-item {
margin-bottom: 8px;
}
}
</style>
@@ -0,0 +1,54 @@
export const stateLabels = {
pending: '等待投递',
processing: '处理中',
retry: '重试等待',
dead: '死信',
delivered: '已投递'
}
export const typeLabels = {
local_event_candidate: '本地事件候选',
audit_projection: '审计投影'
}
export function stateLabel(value) {
return stateLabels[value] || value || '—'
}
export function stateType(value) {
return (
{
pending: 'info',
processing: 'primary',
retry: 'warning',
dead: 'danger',
delivered: 'success'
}[value] || 'info'
)
}
export function typeLabel(value) {
return typeLabels[value] || value || '—'
}
export function buildOutboxQuery(query) {
return {
pageIndex: query.pageIndex,
pageSize: query.pageSize,
state: query.state || undefined,
internalType: query.internalType || undefined,
keyword: String(query.keyword || '').trim() || undefined
}
}
export function nextAction(item, now = Date.now()) {
if (item.state === 'dead') return '等待人工处理'
if (item.state === 'processing') { return item.leaseUntil ? `租约至 ${formatTime(item.leaseUntil)}` : '处理中' }
if (item.state === 'retry') {
return item.availableAt && new Date(item.availableAt).getTime() > now
? `自动重试 ${formatTime(item.availableAt)}`
: '等待重试领取'
}
if (item.state === 'delivered') return '已完成'
return '等待 worker 领取'
}
export function formatTime(value) {
return value
? new Date(value).toLocaleString('zh-CN', { hour12: false })
: '—'
}
@@ -0,0 +1,18 @@
import request from '@/utils/request'
import { acknowledgeOpsAlert, evaluateOpsAlerts, getOpsAlert, listOpsAlerts, recoverOpsAlert } from '@/api/sense/ops-alert'
jest.mock('@/utils/request', () => jest.fn(config => config))
describe('Sense operational alert API', () => {
beforeEach(() => request.mockClear())
test('exposes read, refresh and optimistic state actions', () => {
const query = { pageIndex: 1, pageSize: 10, state: 'unacknowledged' }
const action = { expectedVersion: 3, reason: '已安排现场人员处理设备故障' }
expect(listOpsAlerts(query)).toEqual({ url: '/api/v1/ops-alerts', method: 'get', params: query })
expect(getOpsAlert('alert/a b')).toEqual({ url: '/api/v1/ops-alerts/alert%2Fa%20b', method: 'get' })
expect(evaluateOpsAlerts()).toEqual({ url: '/api/v1/ops-alerts/evaluate', method: 'post' })
expect(acknowledgeOpsAlert('alert/a b', action)).toEqual({ url: '/api/v1/ops-alerts/alert%2Fa%20b/acknowledge', method: 'post', data: action })
expect(recoverOpsAlert('alert/a b', action)).toEqual({ url: '/api/v1/ops-alerts/alert%2Fa%20b/recover', method: 'post', data: action })
})
})
@@ -0,0 +1,25 @@
import { alertStateLabel, alertStateType, alertTypeLabel, buildOpsAlertQuery, canAcknowledge, canRecover } from '@/views/sense/ops-alert/opsAlertState'
describe('Sense operational alert presentation state', () => {
test('uses ordinary operations wording and six alert types', () => {
expect(alertTypeLabel('device_offline')).toBe('设备离线')
expect(alertTypeLabel('authentication_failed')).toBe('认证失败')
expect(alertTypeLabel('clock_drift')).toBe('时间漂移')
expect(alertTypeLabel('reconciliation_failed')).toBe('状态对账失败')
expect(alertTypeLabel('media_shard_failed')).toBe('媒体分片异常')
expect(alertTypeLabel('control_tunnel_failed')).toBe('控制隧道异常')
expect(alertStateLabel('recovering')).toBe('恢复观察')
expect(alertStateType('recovered')).toBe('success')
})
test('enforces the approved state action boundary', () => {
expect(canAcknowledge({ state: 'unacknowledged', version: 1 })).toBe(true)
expect(canAcknowledge({ state: 'acknowledged', version: 2 })).toBe(false)
expect(canRecover({ state: 'recovering', version: 3 })).toBe(true)
expect(canRecover({ state: 'acknowledged', version: 3 })).toBe(false)
})
test('builds an allowlisted query', () => {
expect(buildOpsAlertQuery({ pageIndex: '2', pageSize: 20, alertType: ' clock_drift ', state: ' recovering ', keyword: ' 南门 ', ignored: true })).toEqual({ pageIndex: 2, pageSize: 20, alertType: 'clock_drift', state: 'recovering', keyword: '南门' })
})
})
@@ -0,0 +1,27 @@
import request from '@/utils/request'
import { getOutbox, listOutbox, requeueOutbox } from '@/api/sense/outbox'
jest.mock('@/utils/request', () => jest.fn())
describe('Sense outbox API', () => {
beforeEach(() => request.mockReset())
test('encodes identifiers and sends only the recovery fields', () => {
listOutbox({ pageIndex: 1, pageSize: 10 })
getOutbox('outbox/id unsafe')
requeueOutbox('outbox/id unsafe', 4, '出口故障已排除')
expect(request).toHaveBeenNthCalledWith(1, {
url: '/api/v1/outbox',
method: 'get',
params: { pageIndex: 1, pageSize: 10 }
})
expect(request).toHaveBeenNthCalledWith(2, {
url: '/api/v1/outbox/outbox%2Fid%20unsafe',
method: 'get'
})
expect(request).toHaveBeenNthCalledWith(3, {
url: '/api/v1/outbox/outbox%2Fid%20unsafe/requeue',
method: 'post',
data: { expectedVersion: 4, reason: '出口故障已排除' }
})
})
})
@@ -0,0 +1,37 @@
import {
buildOutboxQuery,
nextAction,
stateLabel,
stateType,
typeLabel
} from '@/views/sense/outbox/outboxState'
describe('Sense outbox presentation state', () => {
test('uses operator-facing labels and non-color-only states', () => {
expect(stateLabel('dead')).toBe('死信')
expect(stateType('dead')).toBe('danger')
expect(typeLabel('local_event_candidate')).toBe('本地事件候选')
})
test('builds an allowlisted trimmed query', () => {
expect(
buildOutboxQuery({
pageIndex: 2,
pageSize: 20,
state: 'retry',
internalType: 'local_event_candidate',
keyword: ' key ',
ignored: 'no'
})
).toEqual({
pageIndex: 2,
pageSize: 20,
state: 'retry',
internalType: 'local_event_candidate',
keyword: 'key'
})
})
test('explains the next recovery action', () => {
expect(nextAction({ state: 'dead' })).toBe('等待人工处理')
expect(nextAction({ state: 'pending' })).toBe('等待 worker 领取')
})
})