feat: 重建 Bell 预警处置生命周期 (#133)
This commit is contained in:
QiuSW
@@ -0,0 +1,172 @@
|
||||
package bell_alert_lifecycle_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
|
||||
adminmodels "go-admin/app/admin/models"
|
||||
"go-admin/app/bell/alert"
|
||||
"go-admin/app/bell/alert_lifecycle"
|
||||
"go-admin/app/bell/event"
|
||||
"go-admin/app/bell/rule"
|
||||
)
|
||||
|
||||
func TestConcurrentLifecycleAndPersistence(t *testing.T) {
|
||||
dsn := os.Getenv("BELL_ALERT_LIFECYCLE_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("integration database not configured")
|
||||
}
|
||||
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
actors := createActors(t, db)
|
||||
service := alert_lifecycle.NewService(db)
|
||||
alertID := createAlert(t, db, "main")
|
||||
const attempts = 20
|
||||
var won atomic.Int32
|
||||
results := make(chan alert_lifecycle.Result, attempts)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < attempts; i++ {
|
||||
wg.Add(1)
|
||||
actor := actors[i%2]
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
result, _ := service.Ack(ctx, alertID, actor)
|
||||
if result.Won {
|
||||
won.Add(1)
|
||||
}
|
||||
results <- result
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
if won.Load() != 1 {
|
||||
t.Fatalf("ack winners=%d", won.Load())
|
||||
}
|
||||
detail, err := service.Get(ctx, alertID)
|
||||
if err != nil || detail.Projection.Status != alert_lifecycle.StatusAcknowledged || len(detail.Timeline) != 1 {
|
||||
t.Fatalf("ack projection=%#v err=%v", detail, err)
|
||||
}
|
||||
winner := actors[0]
|
||||
loser := actors[1]
|
||||
if detail.Projection.AcknowledgedBy == nil || *detail.Projection.AcknowledgedBy != winner.ID {
|
||||
winner, loser = loser, winner
|
||||
}
|
||||
for result := range results {
|
||||
if result.Detail.Projection.AcknowledgedBy != nil && *result.Detail.Projection.AcknowledgedBy != winner.ID {
|
||||
t.Fatal("later ack did not report the true winner")
|
||||
}
|
||||
}
|
||||
if _, err = service.Close(ctx, alertID, alert_lifecycle.CloseInput{Outcome: "site_normal"}, loser); err == nil {
|
||||
t.Fatal("non-owner close succeeded")
|
||||
}
|
||||
if _, err = service.Close(ctx, alertID, alert_lifecycle.CloseInput{}, winner); err == nil {
|
||||
t.Fatal("missing outcome succeeded")
|
||||
}
|
||||
closed, err := service.Close(ctx, alertID, alert_lifecycle.CloseInput{Outcome: "site_normal", Note: "现场正常"}, winner)
|
||||
if err != nil || !closed.Won {
|
||||
t.Fatalf("close failed: %#v %v", closed, err)
|
||||
}
|
||||
replay, err := service.Close(ctx, alertID, alert_lifecycle.CloseInput{Outcome: "site_normal", Note: "现场正常"}, winner)
|
||||
if err != nil || !replay.Idempotent {
|
||||
t.Fatalf("close replay=%#v %v", replay, err)
|
||||
}
|
||||
if _, err = service.Close(ctx, alertID, alert_lifecycle.CloseInput{Outcome: "false_positive"}, winner); err == nil {
|
||||
t.Fatal("conflicting close replay succeeded")
|
||||
}
|
||||
if err = db.Model(&alert_lifecycle.Fact{}).Where("alert_id = ?", alertID).Update("actor_name", "tampered").Error; err == nil {
|
||||
t.Fatal("lifecycle fact update succeeded")
|
||||
}
|
||||
var facts, rejects int64
|
||||
db.Model(&alert_lifecycle.Fact{}).Where("alert_id = ?", alertID).Count(&facts)
|
||||
db.Model(&alert_lifecycle.RejectionAudit{}).Where("alert_id = ?", alertID).Count(&rejects)
|
||||
if facts != 2 || rejects < 20 {
|
||||
t.Fatalf("facts=%d rejects=%d", facts, rejects)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
_ = sqlDB.Close()
|
||||
reopened, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := alert_lifecycle.NewService(reopened).Get(ctx, alertID)
|
||||
if err != nil || after.Projection.Status != alert_lifecycle.StatusClosed || len(after.Timeline) != 2 {
|
||||
t.Fatalf("restart detail=%#v err=%v", after, err)
|
||||
}
|
||||
|
||||
rollbackID := createAlert(t, reopened, "rollback")
|
||||
if err = reopened.Exec(`CREATE FUNCTION bell_test_reject_lifecycle() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'forced lifecycle failure'; END $$`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = reopened.Exec(`CREATE TRIGGER bell_test_reject_lifecycle BEFORE INSERT ON bell_alert_lifecycle_facts FOR EACH ROW EXECUTE FUNCTION bell_test_reject_lifecycle()`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = alert_lifecycle.NewService(reopened).Ack(ctx, rollbackID, winner); err == nil {
|
||||
t.Fatal("forced lifecycle failure succeeded")
|
||||
}
|
||||
rollback, _ := alert_lifecycle.NewService(reopened).Get(ctx, rollbackID)
|
||||
if rollback.Projection.Status != alert_lifecycle.StatusOpen || len(rollback.Timeline) != 0 {
|
||||
t.Fatal("failed ack left partial projection")
|
||||
}
|
||||
if err = reopened.Exec(`DROP TRIGGER bell_test_reject_lifecycle ON bell_alert_lifecycle_facts; DROP FUNCTION bell_test_reject_lifecycle()`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
adminCloseID := createAlert(t, reopened, "admin-close")
|
||||
if _, err = alert_lifecycle.NewService(reopened).Ack(ctx, adminCloseID, winner); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
administrator := loser
|
||||
administrator.Role = "admin"
|
||||
if result, closeErr := alert_lifecycle.NewService(reopened).Close(ctx, adminCloseID, alert_lifecycle.CloseInput{Outcome: "danger_confirmed"}, administrator); closeErr != nil || !result.Won {
|
||||
t.Fatalf("administrator close failed: %#v %v", result, closeErr)
|
||||
}
|
||||
}
|
||||
|
||||
func createActors(t *testing.T, db *gorm.DB) []alert_lifecycle.Actor {
|
||||
t.Helper()
|
||||
var roleID int
|
||||
db.Table("sys_role").Select("role_id").Where("role_key='operator'").Scan(&roleID)
|
||||
result := make([]alert_lifecycle.Actor, 2)
|
||||
for i := range result {
|
||||
user := adminmodels.SysUser{Username: "bell_133_operator_" + string(rune('a'+i)), Password: "test-password-133", NickName: "处置员" + string(rune('A'+i)), RoleId: roleID, DeptId: 1, PostId: 1, Status: "2"}
|
||||
if err := db.Create(&user).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result[i] = alert_lifecycle.Actor{ID: user.UserId, Name: user.NickName, Role: "operator"}
|
||||
}
|
||||
return result
|
||||
}
|
||||
func createAlert(t *testing.T, db *gorm.DB, suffix string) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
eventType := "lifecycle_" + suffix
|
||||
createdRule, err := rule.NewService(db).Create(ctx, rule.WriteInput{Code: "lifecycle-" + suffix, Name: "生命周期规则", EventType: &eventType, MinimumSeverity: "low"}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
created, err := event.NewService(db).Ingest(ctx, event.Command{ProducerID: "bell.lifecycle-test", SourceEventID: suffix, EventType: eventType, OccurredAt: time.Date(2026, 8, 29, 0, 0, 0, 0, time.UTC), Location: "测试地点" + suffix, Severity: "high", Attributes: map[string]any{}}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _, err := alert.NewService(db).List(ctx, alert.PageQuery{PageIndex: 1, PageSize: 100})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, item := range items {
|
||||
if item.PrimaryRuleID == createdRule.ID {
|
||||
return item.ID
|
||||
}
|
||||
}
|
||||
t.Fatal("alert not created")
|
||||
return created.Event.ID
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
[CmdletBinding()] param([string]$PostgresBin='D:\pgsql17\bin')
|
||||
Set-StrictMode -Version 3.0
|
||||
$ErrorActionPreference='Stop'; $started=$false; $server=$null
|
||||
$root=Join-Path ([IO.Path]::GetTempPath()) ('yovision-bell-133-'+[guid]::NewGuid().ToString('N'))
|
||||
$data=Join-Path $root 'postgres'; $log=Join-Path $root 'postgres.log'; $serverRoot=(Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path; $serverExe=Join-Path $root 'bell.exe'
|
||||
function FreePort { $l=[Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback,0); try{$l.Start();return ([Net.IPEndPoint]$l.LocalEndpoint).Port}finally{$l.Stop()} }
|
||||
function WaitPort([int]$port){for($i=0;$i -lt 120;$i++){try{$c=[Net.Sockets.TcpClient]::new();$ok=$c.ConnectAsync('127.0.0.1',$port).Wait(250)-and$c.Connected;$c.Dispose();if($ok){return}}catch{};Start-Sleep -Milliseconds 250};throw 'PostgreSQL did not start'}
|
||||
function Login([string]$base,[string]$username,[string]$password){$body=@{username=$username;password=$password;code='0';uuid='0'}|ConvertTo-Json -Compress; $result=Invoke-RestMethod -Method Post -Uri "$base/api/v1/login" -ContentType 'application/json' -Body $body -NoProxy; if([int]$result.code-ne 200){throw "login failed: $username"}; return @{Authorization="Bearer $($result.token)"}}
|
||||
New-Item -ItemType Directory -Path $root|Out-Null; $port=FreePort
|
||||
try {
|
||||
foreach($name in @('initdb.exe','pg_ctl.exe','createdb.exe','psql.exe')){if(-not(Test-Path (Join-Path $PostgresBin $name))){throw "Missing $name"}}
|
||||
& (Join-Path $PostgresBin 'initdb.exe') -D $data -U postgres -A trust --encoding=UTF8 --no-locale|Out-Null; if($LASTEXITCODE-ne 0){throw 'initdb failed'}
|
||||
$args="-D `"$data`" -l `"$log`" -o `"-p $port -h 127.0.0.1`" start"; Start-Process (Join-Path $PostgresBin 'pg_ctl.exe') -ArgumentList $args -WindowStyle Hidden|Out-Null; WaitPort $port; $started=$true
|
||||
& (Join-Path $PostgresBin 'createdb.exe') -h 127.0.0.1 -p $port -U postgres bell_133; if($LASTEXITCODE-ne 0){throw 'createdb failed'}
|
||||
$bellPort=FreePort; $base="http://127.0.0.1:$bellPort"; $env:GOTOOLCHAIN='go1.26.5'; $env:BELL_DATABASE_URL="host=127.0.0.1 port=$port user=postgres dbname=bell_133 sslmode=disable"; $env:BELL_ALERT_LIFECYCLE_TEST_DATABASE_URL=$env:BELL_DATABASE_URL; $env:BELL_JWT_SECRET=[guid]::NewGuid().ToString('N')+[guid]::NewGuid().ToString('N'); $env:BELL_BOOTSTRAP_USERNAME='bell_133_admin'; $env:BELL_BOOTSTRAP_PASSWORD=[guid]::NewGuid().ToString('N'); $env:BELL_HOST='127.0.0.1'; $env:BELL_PORT=$bellPort.ToString()
|
||||
Push-Location $serverRoot; try { go run . migrate -c config/settings.demo.yml *> (Join-Path $root 'migrate.log'); if($LASTEXITCODE-ne 0){throw "migration failed: $root"}; go test ./tests/bell_alert_lifecycle -count=1 -v; if($LASTEXITCODE-ne 0){throw 'lifecycle test failed'}; go build -o $serverExe . } finally { Pop-Location }
|
||||
$server=Start-Process $serverExe -ArgumentList @('server','-c','config/settings.demo.yml') -WorkingDirectory $serverRoot -RedirectStandardOutput (Join-Path $root 'server.out') -RedirectStandardError (Join-Path $root 'server.err') -WindowStyle Hidden -PassThru; WaitPort $bellPort
|
||||
$a=Login $base 'bell_133_operator_a' 'test-password-133'; $b=Login $base 'bell_133_operator_b' 'test-password-133'; $list=Invoke-RestMethod -Uri "$base/api/v1/bell/alerts?status=open" -Headers $a -NoProxy; $id=[string]$list.data.list[0].id; if([string]::IsNullOrWhiteSpace($id)){throw 'open alert missing'}
|
||||
$ack=Invoke-RestMethod -Method Post -Uri "$base/api/v1/bell/alerts/$id/ack" -Headers $a -ContentType 'application/json' -Body '{}' -NoProxy; $late=Invoke-RestMethod -Method Post -Uri "$base/api/v1/bell/alerts/$id/ack" -Headers $b -ContentType 'application/json' -Body '{}' -NoProxy; if([int]$ack.code-ne 200-or[int]$late.code-ne 409){throw 'ack API semantics failed'}
|
||||
$forbidden=Invoke-RestMethod -Method Post -Uri "$base/api/v1/bell/alerts/$id/close" -Headers $b -ContentType 'application/json' -Body '{"outcome":"site_normal"}' -NoProxy; $missing=Invoke-RestMethod -Method Post -Uri "$base/api/v1/bell/alerts/$id/close" -Headers $a -ContentType 'application/json' -Body '{}' -NoProxy; if([int]$forbidden.code-ne 403-or[int]$missing.code-ne 400){throw 'close rejection semantics failed'}
|
||||
$body='{"outcome":"site_normal","note":"现场正常"}'; $closed=Invoke-RestMethod -Method Post -Uri "$base/api/v1/bell/alerts/$id/close" -Headers $a -ContentType 'application/json; charset=utf-8' -Body $body -NoProxy; $replay=Invoke-RestMethod -Method Post -Uri "$base/api/v1/bell/alerts/$id/close" -Headers $a -ContentType 'application/json; charset=utf-8' -Body $body -NoProxy; $timeline=Invoke-RestMethod -Uri "$base/api/v1/bell/alerts/$id/lifecycle" -Headers $a -NoProxy; if([int]$closed.code-ne 200-or-not$replay.data.idempotent-or$timeline.data.detail.timeline.Count-ne 2){throw 'close/timeline API semantics failed'}
|
||||
$leaks=& (Join-Path $PostgresBin 'psql.exe') -h 127.0.0.1 -p $port -U postgres -d bell_133 -Atc "select count(*) from sys_opera_log where oper_url like '%/bell/alerts/%/close' and ((oper_param <> '' and oper_param not like '%redacted%') or json_result not like '%redacted%');"; if($LASTEXITCODE-ne 0-or[int]$leaks-ne 0){throw 'lifecycle note leaked into GoAdmin operation log'}
|
||||
Write-Output 'BELL_133_HTTP ack=200 late_ack=409 forbidden_close=403 missing_outcome=400 close=200 replay=true timeline=2'
|
||||
} finally {
|
||||
if($null-ne$server-and-not$server.HasExited){Stop-Process -Id $server.Id -Force; $server.WaitForExit(5000)|Out-Null}
|
||||
if($started){& (Join-Path $PostgresBin 'pg_ctl.exe') -D $data -m fast stop *> (Join-Path $root 'stop.log')}
|
||||
foreach($name in @('BELL_DATABASE_URL','BELL_ALERT_LIFECYCLE_TEST_DATABASE_URL','BELL_JWT_SECRET','BELL_BOOTSTRAP_USERNAME','BELL_BOOTSTRAP_PASSWORD','BELL_HOST','BELL_PORT')){Remove-Item "Env:$name" -ErrorAction SilentlyContinue}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package bell_alert_lifecycle_test
|
||||
|
||||
import "testing"
|
||||
|
||||
// Input and state validation are exercised through the PostgreSQL service test;
|
||||
// this sentinel keeps the package runnable without an integration database.
|
||||
func TestLifecyclePackageLoadsWithoutDatabase(t *testing.T) {}
|
||||
@@ -177,7 +177,7 @@ func assertCount(t *testing.T, db *gorm.DB, table string, want int64) {
|
||||
|
||||
func assertOperatorAccess(t *testing.T, db *gorm.DB) {
|
||||
t.Helper()
|
||||
var menuCount, readPolicyCount, writePolicyCount int64
|
||||
var menuCount, readPolicyCount, ruleWritePolicyCount, lifecycleWritePolicyCount int64
|
||||
if err := db.Table("sys_role_menu rm").Joins("JOIN sys_role r ON r.role_id = rm.role_id").
|
||||
Joins("JOIN sys_menu m ON m.menu_id = rm.menu_id").
|
||||
Where("r.role_key = ? AND m.path IN ?", "operator", []string{"/bell", "alerts", "events", "rules"}).Count(&menuCount).Error; err != nil {
|
||||
@@ -186,11 +186,14 @@ func assertOperatorAccess(t *testing.T, db *gorm.DB) {
|
||||
if err := db.Table("casbin_rule").Where("v0 = ? AND v2 = ?", "operator", "GET").Count(&readPolicyCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Table("casbin_rule").Where("v0 = ? AND v2 <> ?", "operator", "GET").Count(&writePolicyCount).Error; err != nil {
|
||||
if err := db.Table("casbin_rule").Where("v0 = ? AND v2 <> ? AND v1 LIKE ?", "operator", "GET", "/api/v1/bell/rules%").Count(&ruleWritePolicyCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if menuCount != 4 || readPolicyCount != 6 || writePolicyCount != 0 {
|
||||
t.Fatalf("operator access is not read-only and minimal: menus=%d reads=%d writes=%d", menuCount, readPolicyCount, writePolicyCount)
|
||||
if err := db.Table("casbin_rule").Where("v0 = ? AND v2 = ? AND v1 IN ?", "operator", "POST", []string{"/api/v1/bell/alerts/:id/ack", "/api/v1/bell/alerts/:id/close"}).Count(&lifecycleWritePolicyCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if menuCount != 4 || readPolicyCount < 6 || ruleWritePolicyCount != 0 || lifecycleWritePolicyCount > 2 {
|
||||
t.Fatalf("operator access escaped Bell scope: menus=%d reads=%d rule_writes=%d lifecycle_writes=%d", menuCount, readPolicyCount, ruleWritePolicyCount, lifecycleWritePolicyCount)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user