diff --git a/docs/13-deployment-and-operations.md b/docs/13-deployment-and-operations.md index 6c4e030..27a6afd 100644 --- a/docs/13-deployment-and-operations.md +++ b/docs/13-deployment-and-operations.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki，禁止直接编辑本文件) wiki_page: Deployment-and-Operations wiki_url: https://git.ilapage.cn/OPC/goauto/wiki/Deployment-and-Operations.- -wiki_revision: 88422ddd1bd239c687b379217e3709f053b77704 -synchronized_at: 2026-09-01T15:31:48Z +wiki_revision: 5b02c5a597ff8c7fa143b0412d6b6426ea17f4a3 +synchronized_at: 2026-09-02T09:29:03Z # 部署与运维 @@ -38,6 +38,11 @@ GOAUTO_ALLOW_INSECURE_AGENT_HTTP=true 5. 真机安装前确认设备没有运行中的任务。Android 系统安装确认仍由人工完成，Agent 不静默安装。 发布、重启、设置当前 Agent 版本和真机安装都属于高风险操作，必须有对应工单和人工确认。不得在部署过程中执行付款、创建订单或清理生产数据。 +### API 长请求超时约束 + +Admin 蝦皮规格 AI 匹配会同步等待外部 Provider：Provider 配置允许 3～600 秒，Admin 客户端等待 610 秒。因此服务端 `settings.application.writetimeout` 必须至少为 620 秒，受版本管理的标准配置固定为 `620`；启动时低于该值必须明确失败，禁止以 2 秒等短写超时运行，否则连接会在领域错误返回前被截断并表现为代理 502。反向代理的读/发送超时也必须大于服务端窗口。 + +Provider 故障日志只允许记录调用关联 ID、操作类型、耗时、上游 HTTP 状态码或网络错误分类；不得记录 API Key、Authorization、Provider URL、模型输入、候选规格、商品原始内容或响应正文。Provider 失败时接口返回 HTTP 503 与 `AI_MATCHING_UNAVAILABLE`，供 Admin 显示安全中文提示。 ## 回滚 diff --git a/server/app/goauto/aimatching/service.go b/server/app/goauto/aimatching/service.go index 4255494..7f8794b 100644 --- a/server/app/goauto/aimatching/service.go +++ b/server/app/goauto/aimatching/service.go @@ -25,12 +25,28 @@ const ( defaultAutoConfirmMinConfidence = 0.9 ) +// MaxProviderTimeout is also the total budget used by composite synchronous +// AI operations. This keeps their HTTP response inside the Admin and API +// transport windows even when an operation needs more than one provider call. +const MaxProviderTimeout = 600 * time.Second + // Service owns the internal AI Provider configuration. The API key exception // is deliberately narrow: it is plain text only in the dedicated settings // table and is returned only by the administrator settings handler. type Service struct { - DB *gorm.DB - HTTPClient *http.Client + DB *gorm.DB + HTTPClient *http.Client + ProviderFailureLogger func(ProviderFailureDiagnostic) +} + +// ProviderFailureDiagnostic deliberately contains no URL, model, prompt, +// candidates, response body or credential. It is safe for operational logs. +type ProviderFailureDiagnostic struct { + CallID string + Operation string + Kind string + StatusCode int + Duration time.Duration } func NewService(db *gorm.DB) *Service { diff --git a/server/app/goauto/aimatching/suggest.go b/server/app/goauto/aimatching/suggest.go index d132b1c..908ea72 100644 --- a/server/app/goauto/aimatching/suggest.go +++ b/server/app/goauto/aimatching/suggest.go @@ -10,6 +10,9 @@ import ( "net/http" "strings" "time" + + log "github.com/go-admin-team/go-admin-core/logger" + "github.com/google/uuid" ) // Suggestion limits are enforced defensively here too, even though callers @@ -95,18 +98,26 @@ func (s *Service) SuggestBatch(ctx context.Context, request SuggestRequest) (Sug } httpRequest.Header.Set("Authorization", "Bearer "+apiKey) httpRequest.Header.Set("Content-Type", "application/json") + callID, startedAt := uuid.NewString(), time.Now() response, err := s.httpClient().Do(httpRequest) if err != nil { + s.logProviderFailure(ProviderFailureDiagnostic{CallID: callID, Operation: "suggest_batch", Kind: providerNetworkErrorKind(err), Duration: time.Since(startedAt)}) return SuggestResult{}, &Error{Code: CodeProviderUnavailable, Message: "AI 建议服务暂时不可用", Cause: err} } defer response.Body.Close() limited := io.LimitReader(response.Body, 1<<20) responseBody, readErr := io.ReadAll(limited) - if readErr != nil || response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + if readErr != nil { + s.logProviderFailure(ProviderFailureDiagnostic{CallID: callID, Operation: "suggest_batch", Kind: "read_error", StatusCode: response.StatusCode, Duration: time.Since(startedAt)}) + return SuggestResult{}, fail(CodeProviderUnavailable, "AI 建议服务暂时不可用") + } + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + s.logProviderFailure(ProviderFailureDiagnostic{CallID: callID, Operation: "suggest_batch", Kind: "http_status", StatusCode: response.StatusCode, Duration: time.Since(startedAt)}) return SuggestResult{}, fail(CodeProviderUnavailable, "AI 建议服务暂时不可用") } raw, err := parseSuggestChoices(responseBody) if err != nil { + s.logProviderFailure(ProviderFailureDiagnostic{CallID: callID, Operation: "suggest_batch", Kind: "invalid_response", StatusCode: response.StatusCode, Duration: time.Since(startedAt)}) return SuggestResult{}, fail(CodeProviderUnavailable, "AI 建议响应无效") } @@ -145,6 +156,22 @@ func (s *Service) SuggestBatch(ctx context.Context, request SuggestRequest) (Sug return SuggestResult{Decisions: decisions, Provider: ProviderOpenAICompatible, Model: setting.Model}, nil } +func (s *Service) logProviderFailure(diagnostic ProviderFailureDiagnostic) { + if s.ProviderFailureLogger != nil { + s.ProviderFailureLogger(diagnostic) + return + } + log.Warnf("AI provider call failed: call_id=%s operation=%s kind=%s status=%d duration_ms=%d", + diagnostic.CallID, diagnostic.Operation, diagnostic.Kind, diagnostic.StatusCode, diagnostic.Duration.Milliseconds()) +} + +func providerNetworkErrorKind(err error) string { + if errors.Is(err, context.DeadlineExceeded) { + return "timeout" + } + return "network_error" +} + func suggestSystemPrompt(dimension string) string { noun := "颜色或尺码" switch dimension { diff --git a/server/app/goauto/aimatching/suggest_test.go b/server/app/goauto/aimatching/suggest_test.go index e2db94b..6eb3d0d 100644 --- a/server/app/goauto/aimatching/suggest_test.go +++ b/server/app/goauto/aimatching/suggest_test.go @@ -6,7 +6,9 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" + "time" "go-admin/app/goauto/models" @@ -134,3 +136,93 @@ func TestSuggestBatchRequiresConfiguredProvider(t *testing.T) { t.Fatalf("expected CodeNotConfigured, got %v", target.Code) } } + +func TestSuggestBatchLogsSafeDiagnosticForProvider502(t *testing.T) { + const sensitiveBody = "api-key-and-provider-body-must-not-be-logged" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, sensitiveBody, http.StatusBadGateway) + })) + defer server.Close() + + db := openSuggestTestDB(t) + seedEnabledSetting(t, db, server.URL) + var diagnostic ProviderFailureDiagnostic + service := NewService(db) + service.ProviderFailureLogger = func(value ProviderFailureDiagnostic) { diagnostic = value } + + _, err := service.SuggestBatch(context.Background(), SuggestRequest{ + Sources: []SuggestSource{{ID: "s1", Label: "sensitive-source"}}, + Candidates: []SuggestCandidate{{ID: "c1", Label: "sensitive-candidate"}}, + }) + if err == nil { + t.Fatal("expected provider failure") + } + if diagnostic.Operation != "suggest_batch" || diagnostic.Kind != "http_status" || diagnostic.StatusCode != http.StatusBadGateway || diagnostic.CallID == "" { + t.Fatalf("unexpected diagnostic: %+v", diagnostic) + } + printed := fmt.Sprintf("%+v", diagnostic) + for _, secret := range []string{sensitiveBody, "sensitive-source", "sensitive-candidate", "test-key", server.URL} { + if strings.Contains(printed, secret) { + t.Fatalf("diagnostic leaked %q: %s", secret, printed) + } + } +} + +func TestSuggestBatchClassifiesProviderTimeout(t *testing.T) { + db := openSuggestTestDB(t) + seedEnabledSetting(t, db, "http://provider.invalid") + var diagnostic ProviderFailureDiagnostic + service := NewService(db) + service.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, context.DeadlineExceeded + })} + service.ProviderFailureLogger = func(value ProviderFailureDiagnostic) { diagnostic = value } + + _, err := service.SuggestBatch(context.Background(), SuggestRequest{ + Sources: []SuggestSource{{ID: "s1", Label: "黑色"}}, Candidates: []SuggestCandidate{{ID: "c1", Label: "黑色"}}, + }) + if err == nil || diagnostic.Kind != "timeout" || diagnostic.StatusCode != 0 { + t.Fatalf("timeout was not safely classified: diagnostic=%+v err=%v", diagnostic, err) + } +} + +func TestSuggestBatchClassifiesInvalidResponseWithoutLoggingBody(t *testing.T) { + const sensitiveBody = "not-json-with-sensitive-provider-details" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(sensitiveBody)) + })) + defer server.Close() + db := openSuggestTestDB(t) + seedEnabledSetting(t, db, server.URL) + var diagnostic ProviderFailureDiagnostic + service := NewService(db) + service.ProviderFailureLogger = func(value ProviderFailureDiagnostic) { diagnostic = value } + + _, err := service.SuggestBatch(context.Background(), SuggestRequest{ + Sources: []SuggestSource{{ID: "s1", Label: "黑色"}}, Candidates: []SuggestCandidate{{ID: "c1", Label: "黑色"}}, + }) + if err == nil || diagnostic.Kind != "invalid_response" || strings.Contains(fmt.Sprintf("%+v", diagnostic), sensitiveBody) { + t.Fatalf("invalid response diagnostic is unsafe or missing: diagnostic=%+v err=%v", diagnostic, err) + } +} + +func TestSuggestBatchAllowsProviderResponseAfterTwoSeconds(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2100 * time.Millisecond) + chatCompletionResponder(`{"suggestions":[{"sourceId":"s1","candidateId":"c1","confidence":0.95,"reason":"match"}]}`)(w, r) + })) + defer server.Close() + + db := openSuggestTestDB(t) + seedEnabledSetting(t, db, server.URL) + result, err := NewService(db).SuggestBatch(context.Background(), SuggestRequest{ + Sources: []SuggestSource{{ID: "s1", Label: "黑色"}}, Candidates: []SuggestCandidate{{ID: "c1", Label: "黑色"}}, + }) + if err != nil || result.Decisions["s1"].CandidateID != "c1" { + t.Fatalf("delayed provider response failed: result=%+v err=%v", result, err) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return fn(request) } diff --git a/server/app/goauto/shopeeproduct/auto_match.go b/server/app/goauto/shopeeproduct/auto_match.go index 91998fe..bbc42e7 100644 --- a/server/app/goauto/shopeeproduct/auto_match.go +++ b/server/app/goauto/shopeeproduct/auto_match.go @@ -48,6 +48,9 @@ type autoMatchRef struct { // before the transaction; the transaction rechecks the complete spec context // and PDD candidate set so a stale decision can never be written. func (service *Service) AutoMatchMappings(ctx context.Context, id uint64, request AutoMatchRequest) (AutoMatchResponse, error) { + ctx, cancel := context.WithTimeout(ctx, aimatching.MaxProviderTimeout) + defer cancel() + requestID := strings.TrimSpace(request.RequestID) if _, err := uuid.Parse(requestID); err != nil { return AutoMatchResponse{}, invalidRequest("requestId 必须是 UUID") diff --git a/server/app/goauto/shopeeproduct/auto_match_test.go b/server/app/goauto/shopeeproduct/auto_match_test.go index 46bfaa1..6ea3d0a 100644 --- a/server/app/goauto/shopeeproduct/auto_match_test.go +++ b/server/app/goauto/shopeeproduct/auto_match_test.go @@ -5,11 +5,13 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "sync/atomic" "testing" "go-admin/app/goauto/models" + "github.com/gin-gonic/gin" "github.com/google/uuid" ) @@ -116,6 +118,20 @@ func TestAutoMatchMappingsProviderFailureDoesNotWrite(t *testing.T) { } } +func TestWriteErrorMapsAIUnavailableToStructured503(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + context, _ := gin.CreateTestContext(recorder) + writeError(context, aiUnavailable("AI 匹配服务暂时不可用，请稍后重试")) + + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), `"code":"AI_MATCHING_UNAVAILABLE"`) { + t.Fatalf("response is not structured: %s", recorder.Body.String()) + } +} + func TestAutoMatchMappingsRejectsContextDriftBeforeAtomicWrite(t *testing.T) { db := openTestDB(t) pdd := seedPDDProduct(t, db, "active") diff --git a/server/cmd/api/server.go b/server/cmd/api/server.go index e6c1613..3266e0c 100644 --- a/server/cmd/api/server.go +++ b/server/cmd/api/server.go @@ -54,6 +54,12 @@ var ( } ) +// The synchronous Admin AI endpoints allow a provider timeout of up to 600 +// seconds and the browser waits 610 seconds. Keep the HTTP server alive a +// little longer so it can return the domain response instead of truncating +// the connection and surfacing a proxy-level 502. +const minimumAPIWriteTimeout = 620 * time.Second + var AppRouters = make([]func(), 0) func init() { @@ -125,11 +131,15 @@ func run() error { ) } + writeTimeout, err := validatedAPIWriteTimeout(config.ApplicationConfig.WriterTimeout) + if err != nil { + return err + } srv := &http.Server{ Addr: fmt.Sprintf("%s:%d", config.ApplicationConfig.Host, config.ApplicationConfig.Port), Handler: sdk.Runtime.GetEngine(), ReadTimeout: time.Duration(config.ApplicationConfig.ReadTimeout) * time.Second, - WriteTimeout: time.Duration(config.ApplicationConfig.WriterTimeout) * time.Second, + WriteTimeout: writeTimeout, } go func() { @@ -193,6 +203,14 @@ func run() error { return nil } +func validatedAPIWriteTimeout(seconds int) (time.Duration, error) { + timeout := time.Duration(seconds) * time.Second + if timeout < minimumAPIWriteTimeout { + return 0, fmt.Errorf("application writetimeout must be at least %s for synchronous AI requests", minimumAPIWriteTimeout) + } + return timeout, nil +} + type policyLoader interface { LoadPolicy() error } diff --git a/server/cmd/api/server_timeout_test.go b/server/cmd/api/server_timeout_test.go new file mode 100644 index 0000000..5034b6e --- /dev/null +++ b/server/cmd/api/server_timeout_test.go @@ -0,0 +1,19 @@ +package api + +import ( + "testing" + "time" +) + +func TestValidatedAPIWriteTimeoutProtectsSynchronousAIRequests(t *testing.T) { + if _, err := validatedAPIWriteTimeout(2); err == nil { + t.Fatal("two-second write timeout must be rejected") + } + got, err := validatedAPIWriteTimeout(620) + if err != nil { + t.Fatalf("620-second write timeout should be accepted: %v", err) + } + if got != 620*time.Second { + t.Fatalf("write timeout = %s, want 620s", got) + } +} diff --git a/server/config/settings.full.yml b/server/config/settings.full.yml index bcca7e2..3722d2a 100644 --- a/server/config/settings.full.yml +++ b/server/config/settings.full.yml @@ -9,7 +9,7 @@ settings: # 端口号 port: 8000 # 服务端口号 readtimeout: 1 - writertimeout: 2 + writertimeout: 620 # 数据权限功能开关 enabledp: false ssl: diff --git a/server/config/settings.yml b/server/config/settings.yml index 0533615..9c0a32e 100644 --- a/server/config/settings.yml +++ b/server/config/settings.yml @@ -9,7 +9,7 @@ settings: # 端口号 port: 8000 # 服务端口号 readtimeout: 1 - writertimeout: 2 + writertimeout: 620 # 数据权限功能开关 enabledp: false logger: @@ -84,4 +84,4 @@ settings: # blockingTimeout: 5 # reclaimInterval: 1 locker: - redis: \ No newline at end of file + redis: