diff --git a/.gitignore b/.gitignore index 8861aa2..94b4995 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ __pycache__/ # Gitea 凭据,禁止提交 gitea.env + +# Local Portal runtime settings may contain deployment-specific values. +/portal/config/settings.yml diff --git a/docs/00-project-profile.md b/docs/00-project-profile.md index 6d982b0..41cba84 100644 --- a/docs/00-project-profile.md +++ b/docs/00-project-profile.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件) wiki_page: Project-Profile wiki_url: https://git.ilapage.cn/OPC/chorus/wiki/Project-Profile.- -wiki_revision: 5392ab80c7789a793ee271c85d2aca875bedc7f2 -synchronized_at: 2026-08-24T07:59:16Z +wiki_revision: cd93968f7c48ebbe0a594f6198c29726e8ac9c3e +synchronized_at: 2026-08-25T07:00:30Z # 项目档案 @@ -175,13 +175,13 @@ synchronized_at: 2026-08-24T07:59:16Z Windows 本地开发可使用 `config/local-services.yml` 与 `scripts/start-all.bat`、`status-all.bat`、`stop-all.bat` 统一管理 Portal、Admin API 和 Admin UI。统一配置只保存 loopback Host、端口和受保护配置文件路径,不保存或复制 DSN、JWT secret、密码、Token 或 Provider Key。 -Portal 的敏感配置仍来自 `portal.environment_file` 指向的受保护 PowerShell 环境文件,统一配置只覆盖 `CHORUS_LISTEN_ADDRESS`;Admin API 的 Host/Port 和敏感值仍以 `admin.settings_file` 指向的 settings.yml 为唯一事实源;Admin UI Host/Port 来自统一配置,API 基址在启动时由 Admin 地址派生。脚本不会运行 migration、AutoMigrate、seed、管理员 bootstrap 或依赖安装。 +Portal 的数据库、Session 等敏感配置仍来自 `portal.environment_file` 指向的受保护 PowerShell 环境文件;Provider/worker 运维参数来自 `portal.settings_file` 指向的严格 YAML,已设置的同名环境变量逐项覆盖 YAML。统一配置覆盖 `CHORUS_LISTEN_ADDRESS` 并传递两个 Portal 配置路径;Admin API 的 Host/Port 和敏感值仍以 `admin.settings_file` 为唯一事实源。脚本不会运行 migration、AutoMigrate、seed、管理员 bootstrap 或依赖安装。 统一脚本只管理状态文件记录且可执行路径匹配的进程。端口被未受管进程占用时显示 `unmanaged listener` 并拒绝接管;启动中途失败只回滚本次新启动的组件。现有 `scripts/chorus-dev.bat` 继续用于单独管理 Portal 和 mock。 ## #39 本机 Supervisor 三实例(2026-08-24,验收通过) -本机 `D:\supervisor` 通过 `scripts/install-supervisor.bat` 安装并托管三个独立实例:`chorus-user`、`chorus-admin-api`、`chorus-admin-ui`。Portal 和 Admin UI 地址来自 `config/local-services.yml`;Admin API 地址与运行配置来自 `admin/config/settings.yml`。安装器只把构建产物和不含凭据的 program 配置写入 Supervisor 目录,Portal 启动时才在进程内读取受保护环境文件。 +本机 `D:\supervisor` 通过 `scripts/install-supervisor.bat` 安装并托管三个独立实例:`chorus-user`、`chorus-admin-api`、`chorus-admin-ui`。Portal 地址、环境文件和 YAML settings 路径来自 `config/local-services.yml`;Admin API 地址与运行配置来自 `admin/config/settings.yml`。安装器只把构建产物和配置路径写入 Supervisor 目录,Portal 启动时才在进程内读取受保护文件。 三个实例均以前台子进程运行,启用自动重启、独立日志和进程组停止。更新代码或统一配置后重新运行安装器,再执行 Supervisor `reload`。此入口只用于当前 Windows 本地环境,不执行 migration、AutoMigrate、seed 或账号创建。 diff --git a/docs/02-architecture-and-code-map.md b/docs/02-architecture-and-code-map.md index 716b873..5f33cd0 100644 --- a/docs/02-architecture-and-code-map.md +++ b/docs/02-architecture-and-code-map.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件) wiki_page: Architecture-and-Code-Map wiki_url: https://git.ilapage.cn/OPC/chorus/wiki/Architecture-and-Code-Map.- -wiki_revision: 84986fd08a4814ac64fd61d7ab5c02a06d815dc8 -synchronized_at: 2026-08-25T03:56:20Z +wiki_revision: 451c402646bbd8431a12b624cbe42072d40aa1ca +synchronized_at: 2026-08-25T07:00:45Z # 架构与代码地图 @@ -106,7 +106,7 @@ MVP-0 第一张上传图自动作为 `primary`,其余为 `reference`;不提 - MySQL 8 集成测试分别完成一次 chat 和 images_edits,断言 rendered_prompt、attempt ProviderModel/latency、输出记录、原图和缩略图;全程不访问真实 Provider。 ### #15 portal 组合根与运行闭环 -- `portal/main.go` 现在实际组装 MySQL queue controller、SSRF 安全 HTTP client、Provider factory/catalog、本地存储和内嵌 worker;同步 handler 仍只写 pending,不持有 Provider 依赖。 +- `portal/main.go` 现在实际组装 MySQL queue controller、SSRF 安全 HTTP client、Provider factory/catalog、本地存储和内嵌 worker;同步 handler 仍只写 pending,不持有 Provider 依赖。Portal 启动必须显式传入 `--config `;Provider/worker 参数从严格 YAML 读取,同名环境变量优先覆盖。 - worker 的 claim 必须经过 `queue.Controller`;收到退出信号后先阻止新 claim,HTTP server 停止接收请求,并等待在途 worker 在租约上下文内完成。HTTP 或 worker 任一组件意外结束都会让进程失败关闭。 - 非生产默认 lease 60 秒、Provider HTTP 超时 45 秒、轮询 250ms、响应上限 32MiB;生产必须显式配置,且 lease 必须比 HTTP 超时多 5 秒以上。 - `CHORUS_TEST_DISABLE_WORKER=true` 只允许 `CHORUS_ENV=test`,用于浏览器稳定观察状态的测试 fixture;开发和生产环境启用会直接报错。真实 Provider/worker 成功仍通过注入受控 DNS/DialContext 的 MySQL 集成测试验证,生产 SSRF 逻辑没有旁路。 diff --git a/docs/04-local-development-and-verification.md b/docs/04-local-development-and-verification.md index e0e7eb7..ac919fb 100644 --- a/docs/04-local-development-and-verification.md +++ b/docs/04-local-development-and-verification.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件) wiki_page: Local-Development-and-Verification wiki_url: https://git.ilapage.cn/OPC/chorus/wiki/Local-Development-and-Verification.- -wiki_revision: 9931fb7acc36ebcbcc122bcbd528887b970efb10 -synchronized_at: 2026-08-25T03:56:31Z +wiki_revision: 660d4bc4e857a92c6cf6606d2af95d1b5f2f6d04 +synchronized_at: 2026-08-25T07:00:57Z # 本地开发与验证 @@ -202,14 +202,14 @@ scripts\chorus-dev.bat restart scripts\chorus-dev.bat stop ``` -`start` 会校验环境文件、本地 MySQL TCP 连接和监听端口,构建临时可执行文件,在后台启动 portal,并检查 `/login`。重复启动不会创建第二个 portal;`restart` 只重启 portal;`stop` 只停止由脚本记录且可执行文件路径匹配的 portal 和 mock 进程。脚本不会执行迁移、GORM AutoMigrate、seed 或创建测试账号。 +`start` 会校验环境文件、`portal/config/settings.yml`、本地 MySQL TCP 连接和监听端口,构建临时可执行文件,以 `--config` 显式传入 YAML 后在后台启动 portal,并检查 `/login`。重复启动不会创建第二个 portal;`restart` 只重启 portal;`stop` 只停止由脚本记录且可执行文件路径匹配的 portal 和 mock 进程。脚本不会执行迁移、GORM AutoMigrate、seed 或创建测试账号。 脚本默认读取仓库外的 `D:\OPC\chorus-tools\chorus-test.env.ps1`;也可把其他可信环境文件作为带引号的第二个参数传入,例如 `scripts\chorus-dev.bat start "D:\tools\chorus local.env.ps1"`。日志、PID 状态和临时可执行文件位于 `%LOCALAPPDATA%\Chorus\dev`。 需要在当前终端观察输出时,仍可运行: ```powershell -go run ./portal +go run ./portal --config portal/config/settings.yml ``` 浏览器验证: @@ -361,6 +361,7 @@ portal: host: 127.0.0.1 port: 8080 environment_file: ../../chorus-tools/chorus-test.env.ps1 + settings_file: ../portal/config/settings.yml admin: settings_file: ../admin/config/settings.yml admin_ui: @@ -368,7 +369,7 @@ admin_ui: port: 9528 ``` -相对路径以 `local-services.yml` 所在目录为基准。配置严格拒绝未知字段、非 loopback Host、无效/重复端口、缺失文件或错误扩展名。`chorus-local-config` 读取 Admin settings 时只投影 `settings.application.host/port`,不会输出其他配置内容。Admin Host/Port 不在统一配置重复保存,避免两个事实源漂移。 +相对路径以 `local-services.yml` 所在目录为基准。Portal `environment_file` 保存数据库、Session 等敏感环境变量;`settings_file` 保存 Provider/worker 运维参数,环境变量优先覆盖 YAML。配置严格拒绝未知字段、非 loopback Host、无效/重复端口、缺失文件或错误扩展名。`chorus-local-config` 读取 Admin settings 时只投影 `settings.application.host/port`,不会输出其他配置内容。Admin Host/Port 不在统一配置重复保存,避免两个事实源漂移。 从仓库根目录执行: @@ -398,7 +399,7 @@ D:\supervisor\supervisord.exe ctl /c D:\supervisor\supervisord.conf status | Supervisor 实例 | 组件 | 地址来源 | |---|---|---| -| `chorus-user` | Portal 与内嵌 worker | `config/local-services.yml` 的 `portal` | +| `chorus-user` | Portal 与内嵌 worker | `config/local-services.yml` 的 `portal` 地址、环境文件和 settings 文件 | | `chorus-admin-api` | Admin API | `admin/config/settings.yml` 的 `settings.application` | | `chorus-admin-ui` | Vue 开发服务 | `config/local-services.yml` 的 `admin_ui`;API 基址由 Admin 地址派生 | diff --git a/docs/06-troubleshooting.md b/docs/06-troubleshooting.md index 959a153..fbd233e 100644 --- a/docs/06-troubleshooting.md +++ b/docs/06-troubleshooting.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件) wiki_page: Troubleshooting wiki_url: https://git.ilapage.cn/OPC/chorus/wiki/Troubleshooting -wiki_revision: 47ed912393c434b5547bff7ee4a3c488af03b749 -synchronized_at: 2026-08-25T03:56:42Z +wiki_revision: 77dbfc1946d599f06a69afc204297dbe198a0cff +synchronized_at: 2026-08-25T07:01:14Z # 故障排查 @@ -69,6 +69,8 @@ LIMIT 10; 5. 环境代理是否绕开安全 Dial; 6. 上游返回的结果 URL 是否复用相同客户端。 +若 attempt latency 精确接近 Provider HTTP timeout,先检查 `portal/config/settings.yml` 的 `provider.http_timeout_seconds`;同时保证 `worker.lease_seconds` 比它大 5 秒以上。修改后必须重启 Portal,页面中的 ProviderModel timeout 不能突破更短的全局 HTTP timeout。 + 拦截私网是正确行为,不能为联调关闭。需要访问内部 mock 时把 mock 与测试进程放在专用测试网络,并使用测试专用注入点,不在生产配置放行私网。 ### 6. 文件和权限 diff --git a/docs/10-deployment-and-operations.md b/docs/10-deployment-and-operations.md index c12f46b..eb2f422 100644 --- a/docs/10-deployment-and-operations.md +++ b/docs/10-deployment-and-operations.md @@ -2,8 +2,8 @@ generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件) wiki_page: Deployment-and-Operations wiki_url: https://git.ilapage.cn/OPC/chorus/wiki/Deployment-and-Operations.- -wiki_revision: 62fb1a6eb027824fce0d4147e6afba7c06e68fcf -synchronized_at: 2026-08-25T03:57:04Z +wiki_revision: f75c6883edc8dc431fc04cac1dee22aa2515e0d5 +synchronized_at: 2026-08-25T07:01:43Z # 部署与运维 @@ -119,7 +119,7 @@ MVP-0 的终端用户会话保存在单个 portal 进程内存中,Cookie 只 上传先在应用层校验,再在 generation 事务回调中保存带 owner/generation metadata 的文件。事务失败只清理本请求保存的 key;存储根目录不能作为 nginx/static 目录暴露。 ## MVP-0 worker 已实现配置与退出行为 -portal 单二进制会同时启动 HTTP server 和内嵌 worker。生产除现有 portal 限制外,还必须显式提供: +portal 单二进制会同时启动 HTTP server 和内嵌 worker。启动必须显式传入 `--config `。下列 Provider/worker 参数以 YAML 为基础值,同名环境变量可逐项覆盖: | 变量 | 约束 | |---|---| @@ -197,7 +197,7 @@ Provider API Key 由授权管理员通过管理端写入 `provider_credentials.a ## Windows 本机 Supervisor 运行 -`D:\supervisor` 是当前 Windows 本地常驻运行方式,不替代生产反向代理或正式服务管理器。通过 `scripts/install-supervisor.bat` 生成三个独立实例:Portal、Admin API 和 Admin UI。Supervisor program 配置只保存进程路径、配置路径、Host/Port 与日志策略;Portal 凭据继续从 `portal.environment_file` 指向的受保护 PowerShell 文件读取,Admin 敏感值继续只存在于 `admin/config/settings.yml`。 +`D:\supervisor` 是当前 Windows 本地常驻运行方式,不替代生产反向代理或正式服务管理器。通过 `scripts/install-supervisor.bat` 生成三个独立实例:Portal、Admin API 和 Admin UI。Supervisor program 配置只保存进程路径、配置路径、Host/Port 与日志策略;Portal 数据库/Session 配置从 `portal.environment_file` 读取,Provider/worker 参数从 `portal.settings_file` 读取,Admin 敏感值继续只存在于 `admin/config/settings.yml`。 安装或更新: diff --git a/internal/config/config.go b/internal/config/config.go index b6fcf2a..f3e9cc3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -53,6 +53,22 @@ func Load() (Config, error) { } func LoadFromLookup(lookup func(string) (string, bool)) (Config, error) { + return loadFromLookup(lookup, defaultRuntimeSettings()) +} + +func LoadFile(path string) (Config, error) { + return LoadFileFromLookup(path, os.LookupEnv) +} + +func LoadFileFromLookup(path string, lookup func(string) (string, bool)) (Config, error) { + settings, err := loadRuntimeSettings(path) + if err != nil { + return Config{}, err + } + return loadFromLookup(lookup, settings) +} + +func loadFromLookup(lookup func(string) (string, bool), defaults runtimeSettings) (Config, error) { read := func(name string) string { value, _ := lookup(name) return strings.TrimSpace(value) @@ -102,7 +118,7 @@ func LoadFromLookup(lookup func(string) (string, bool)) (Config, error) { missing = append(missing, name) } } - for _, name := range []string{"CHORUS_SESSION_TTL_MINUTES", "CHORUS_LOGIN_ATTEMPTS", "CHORUS_LOGIN_WINDOW_SECONDS", "CHORUS_MAX_PROMPT_BYTES", "CHORUS_MAX_IMAGES", "CHORUS_MAX_IMAGE_BYTES", "CHORUS_MAX_UPLOAD_BYTES", "CHORUS_MAX_IMAGE_PIXELS", "CHORUS_HISTORY_LIMIT", "CHORUS_WORKER_LEASE_SECONDS", "CHORUS_WORKER_POLL_MILLISECONDS", "CHORUS_PROVIDER_HTTP_TIMEOUT_SECONDS", "CHORUS_PROVIDER_MAX_RESPONSE_BYTES", "CHORUS_USER_RATE_LIMIT_CAPACITY", "CHORUS_USER_RATE_LIMIT_WINDOW_SECONDS", "CHORUS_API_KEY_RATE_LIMIT_CAPACITY", "CHORUS_API_KEY_RATE_LIMIT_WINDOW_SECONDS", "CHORUS_PROVIDER_RATE_LIMIT_CAPACITY", "CHORUS_PROVIDER_RATE_LIMIT_WINDOW_SECONDS"} { + for _, name := range []string{"CHORUS_SESSION_TTL_MINUTES", "CHORUS_LOGIN_ATTEMPTS", "CHORUS_LOGIN_WINDOW_SECONDS", "CHORUS_MAX_PROMPT_BYTES", "CHORUS_MAX_IMAGES", "CHORUS_MAX_IMAGE_BYTES", "CHORUS_MAX_UPLOAD_BYTES", "CHORUS_MAX_IMAGE_PIXELS", "CHORUS_HISTORY_LIMIT", "CHORUS_USER_RATE_LIMIT_CAPACITY", "CHORUS_USER_RATE_LIMIT_WINDOW_SECONDS", "CHORUS_API_KEY_RATE_LIMIT_CAPACITY", "CHORUS_API_KEY_RATE_LIMIT_WINDOW_SECONDS", "CHORUS_PROVIDER_RATE_LIMIT_CAPACITY", "CHORUS_PROVIDER_RATE_LIMIT_WINDOW_SECONDS"} { if read(name) == "" { missing = append(missing, name) } @@ -145,22 +161,26 @@ func LoadFromLookup(lookup func(string) (string, bool)) (Config, error) { if cfg.HistoryLimit, err = intConfig(read("CHORUS_HISTORY_LIMIT"), 50); err != nil { return Config{}, fmt.Errorf("CHORUS_HISTORY_LIMIT is invalid") } - if cfg.WorkerLeaseDuration, err = durationConfig(read("CHORUS_WORKER_LEASE_SECONDS"), 60, time.Second); err != nil { + if cfg.WorkerLeaseDuration, err = durationConfig(read("CHORUS_WORKER_LEASE_SECONDS"), defaults.Worker.LeaseSeconds, time.Second); err != nil { return Config{}, fmt.Errorf("CHORUS_WORKER_LEASE_SECONDS is invalid") } - if cfg.WorkerPollInterval, err = durationConfig(read("CHORUS_WORKER_POLL_MILLISECONDS"), 250, time.Millisecond); err != nil { + if cfg.WorkerPollInterval, err = durationConfig(read("CHORUS_WORKER_POLL_MILLISECONDS"), defaults.Worker.PollMilliseconds, time.Millisecond); err != nil { return Config{}, fmt.Errorf("CHORUS_WORKER_POLL_MILLISECONDS is invalid") } - if cfg.ProviderHTTPTimeout, err = durationConfig(read("CHORUS_PROVIDER_HTTP_TIMEOUT_SECONDS"), 45, time.Second); err != nil { + if cfg.ProviderHTTPTimeout, err = durationConfig(read("CHORUS_PROVIDER_HTTP_TIMEOUT_SECONDS"), defaults.Provider.HTTPTimeoutSeconds, time.Second); err != nil { return Config{}, fmt.Errorf("CHORUS_PROVIDER_HTTP_TIMEOUT_SECONDS is invalid") } - providerMaxResponseBytes, err := intConfig(read("CHORUS_PROVIDER_MAX_RESPONSE_BYTES"), 32<<20) + providerMaxResponseBytes, err := intConfig(read("CHORUS_PROVIDER_MAX_RESPONSE_BYTES"), defaults.Provider.MaxResponseBytes) if err != nil { return Config{}, fmt.Errorf("CHORUS_PROVIDER_MAX_RESPONSE_BYTES is invalid") } cfg.ProviderMaxResponseBytes = int64(providerMaxResponseBytes) - if cfg.ProviderAllowedPorts, err = ParseProviderAllowedPorts(read("CHORUS_PROVIDER_ALLOWED_PORTS")); err != nil { - return Config{}, fmt.Errorf("CHORUS_PROVIDER_ALLOWED_PORTS is invalid") + if raw := read("CHORUS_PROVIDER_ALLOWED_PORTS"); raw != "" { + if cfg.ProviderAllowedPorts, err = ParseProviderAllowedPorts(raw); err != nil { + return Config{}, fmt.Errorf("CHORUS_PROVIDER_ALLOWED_PORTS is invalid") + } + } else { + cfg.ProviderAllowedPorts = append([]uint16(nil), defaults.Provider.AllowedPorts...) } userRateCapacity, err := intConfig(read("CHORUS_USER_RATE_LIMIT_CAPACITY"), 60) if err != nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e1baa24..41a8a1f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,11 +1,89 @@ package config import ( + "os" + "path/filepath" "strings" "testing" "time" ) +func TestLoadFileUsesYAMLAndEnvironmentOverrides(t *testing.T) { + path := writeSettings(t, `schema_version: 1 +provider: + http_timeout_seconds: 120 + max_response_bytes: 1048576 + allowed_ports: [80, 443, 8080, 8080] +worker: + lease_seconds: 150 + poll_milliseconds: 500 +`) + cfg, err := LoadFileFromLookup(path, lookup(map[string]string{"CHORUS_DSN": "test-dsn"})) + if err != nil { + t.Fatal(err) + } + if cfg.ProviderHTTPTimeout != 120*time.Second || cfg.ProviderMaxResponseBytes != 1048576 || cfg.WorkerLeaseDuration != 150*time.Second || cfg.WorkerPollInterval != 500*time.Millisecond { + t.Fatalf("YAML settings were not loaded: %#v", cfg) + } + if got := cfg.ProviderAllowedPorts; len(got) != 3 || got[2] != 8080 { + t.Fatalf("provider ports = %v", got) + } + + overrides := map[string]string{ + "CHORUS_DSN": "test-dsn", + "CHORUS_PROVIDER_HTTP_TIMEOUT_SECONDS": "60", + "CHORUS_PROVIDER_MAX_RESPONSE_BYTES": "2097152", + "CHORUS_PROVIDER_ALLOWED_PORTS": "80,443,8443", + "CHORUS_WORKER_LEASE_SECONDS": "90", + "CHORUS_WORKER_POLL_MILLISECONDS": "100", + } + cfg, err = LoadFileFromLookup(path, lookup(overrides)) + if err != nil { + t.Fatal(err) + } + if cfg.ProviderHTTPTimeout != 60*time.Second || cfg.ProviderMaxResponseBytes != 2097152 || cfg.WorkerLeaseDuration != 90*time.Second || cfg.WorkerPollInterval != 100*time.Millisecond || cfg.ProviderAllowedPorts[2] != 8443 { + t.Fatalf("environment overrides were not applied: %#v", cfg) + } +} + +func TestLoadFileRejectsInvalidSettings(t *testing.T) { + tests := map[string]string{ + "unknown field": `schema_version: 1 +provider: {http_timeout_seconds: 45, max_response_bytes: 1024, allowed_ports: [80]} +worker: {lease_seconds: 60, poll_milliseconds: 250} +unknown: true +`, + "schema": `schema_version: 2 +provider: {http_timeout_seconds: 45, max_response_bytes: 1024, allowed_ports: [80]} +worker: {lease_seconds: 60, poll_milliseconds: 250} +`, + "empty ports": `schema_version: 1 +provider: {http_timeout_seconds: 45, max_response_bytes: 1024, allowed_ports: []} +worker: {lease_seconds: 60, poll_milliseconds: 250} +`, + "unsafe lease": `schema_version: 1 +provider: {http_timeout_seconds: 60, max_response_bytes: 1024, allowed_ports: [80]} +worker: {lease_seconds: 65, poll_milliseconds: 250} +`, + } + for name, body := range tests { + t.Run(name, func(t *testing.T) { + if _, err := LoadFileFromLookup(writeSettings(t, body), lookup(map[string]string{"CHORUS_DSN": "test-dsn"})); err == nil { + t.Fatal("invalid settings were accepted") + } + }) + } +} + +func writeSettings(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "settings.yml") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path +} + func lookup(values map[string]string) func(string) (string, bool) { return func(key string) (string, bool) { value, ok := values[key] diff --git a/internal/config/file.go b/internal/config/file.go new file mode 100644 index 0000000..25bc38f --- /dev/null +++ b/internal/config/file.go @@ -0,0 +1,117 @@ +package config + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/goccy/go-yaml" +) + +const maxSettingsBytes = 1 << 20 + +type runtimeSettings struct { + SchemaVersion int `yaml:"schema_version"` + Provider providerSettings `yaml:"provider"` + Worker workerSettings `yaml:"worker"` +} + +type providerSettings struct { + HTTPTimeoutSeconds int `yaml:"http_timeout_seconds"` + MaxResponseBytes int `yaml:"max_response_bytes"` + AllowedPorts []uint16 `yaml:"allowed_ports"` +} + +type workerSettings struct { + LeaseSeconds int `yaml:"lease_seconds"` + PollMilliseconds int `yaml:"poll_milliseconds"` +} + +func defaultRuntimeSettings() runtimeSettings { + return runtimeSettings{ + SchemaVersion: 1, + Provider: providerSettings{ + HTTPTimeoutSeconds: 45, + MaxResponseBytes: 32 << 20, + AllowedPorts: []uint16{80, 443}, + }, + Worker: workerSettings{LeaseSeconds: 60, PollMilliseconds: 250}, + } +} + +func loadRuntimeSettings(path string) (runtimeSettings, error) { + if strings.TrimSpace(path) == "" { + return runtimeSettings{}, errors.New("portal --config is required") + } + extension := strings.ToLower(filepath.Ext(path)) + if extension != ".yml" && extension != ".yaml" { + return runtimeSettings{}, errors.New("portal settings file must use .yml or .yaml") + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() { + return runtimeSettings{}, errors.New("portal settings file does not exist") + } + if info.Size() > maxSettingsBytes { + return runtimeSettings{}, errors.New("portal settings file exceeds 1 MiB") + } + body, err := os.ReadFile(path) + if err != nil { + return runtimeSettings{}, errors.New("read portal settings file") + } + var settings runtimeSettings + decoder := yaml.NewDecoder(bytes.NewReader(body), yaml.Strict()) + if err := decoder.Decode(&settings); err != nil { + return runtimeSettings{}, fmt.Errorf("portal settings file is invalid: %w", err) + } + if err := rejectAdditionalDocument(decoder); err != nil { + return runtimeSettings{}, err + } + if settings.SchemaVersion != 1 { + return runtimeSettings{}, errors.New("portal settings schema_version must be 1") + } + if settings.Provider.HTTPTimeoutSeconds <= 0 || settings.Provider.MaxResponseBytes <= 0 || settings.Worker.LeaseSeconds <= 0 || settings.Worker.PollMilliseconds <= 0 { + return runtimeSettings{}, errors.New("portal settings values must be positive") + } + ports, err := normalizePorts(settings.Provider.AllowedPorts) + if err != nil { + return runtimeSettings{}, err + } + settings.Provider.AllowedPorts = ports + if settings.Worker.LeaseSeconds <= settings.Provider.HTTPTimeoutSeconds+5 { + return runtimeSettings{}, errors.New("worker lease must exceed provider HTTP timeout by more than 5 seconds") + } + return settings, nil +} + +func rejectAdditionalDocument(decoder *yaml.Decoder) error { + var extra any + if err := decoder.Decode(&extra); err == io.EOF { + return nil + } else if err != nil { + return fmt.Errorf("portal settings file is invalid: %w", err) + } + return errors.New("portal settings file must contain one YAML document") +} + +func normalizePorts(values []uint16) ([]uint16, error) { + if len(values) == 0 { + return nil, errors.New("provider allowed_ports must not be empty") + } + ports := make([]uint16, 0, len(values)) + seen := make(map[uint16]struct{}, len(values)) + for _, port := range values { + if port == 0 { + return nil, errors.New("provider allowed_ports contains an invalid port") + } + if _, exists := seen[port]; exists { + continue + } + seen[port] = struct{}{} + ports = append(ports, port) + } + return ports, nil +} diff --git a/internal/localservices/config.go b/internal/localservices/config.go index 01e8569..e58e32a 100644 --- a/internal/localservices/config.go +++ b/internal/localservices/config.go @@ -25,6 +25,7 @@ type PortalConfig struct { Host string `yaml:"host"` Port int `yaml:"port"` EnvironmentFile string `yaml:"environment_file"` + SettingsFile string `yaml:"settings_file"` } type AdminConfig struct { @@ -39,6 +40,7 @@ type EndpointConfig struct { type Resolved struct { PortalAddress string `json:"portal_address"` PortalEnvironmentFile string `json:"portal_environment_file"` + PortalSettingsFile string `json:"portal_settings_file"` AdminAddress string `json:"admin_address"` AdminSettingsFile string `json:"admin_settings_file"` AdminUIAddress string `json:"admin_ui_address"` @@ -80,6 +82,10 @@ func Load(path string) (Resolved, error) { if err != nil { return Resolved{}, err } + portalSettingsFile, err := resolveReferencedFile(baseDir, config.Portal.SettingsFile, "portal.settings_file", ".yml", ".yaml") + if err != nil { + return Resolved{}, err + } settingsFile, err := resolveReferencedFile(baseDir, config.Admin.SettingsFile, "admin.settings_file", ".yml", ".yaml") if err != nil { return Resolved{}, err @@ -102,6 +108,7 @@ func Load(path string) (Resolved, error) { return Resolved{ PortalAddress: portalAddress, PortalEnvironmentFile: environmentFile, + PortalSettingsFile: portalSettingsFile, AdminAddress: adminAddress, AdminSettingsFile: settingsFile, AdminUIAddress: adminUIAddress, diff --git a/internal/localservices/config_test.go b/internal/localservices/config_test.go index bc8ce07..a8ded17 100644 --- a/internal/localservices/config_test.go +++ b/internal/localservices/config_test.go @@ -10,6 +10,7 @@ import ( func TestLoadResolvesOnlyNonSensitiveRuntimeValues(t *testing.T) { dir := t.TempDir() writeTestFile(t, filepath.Join(dir, "portal.ps1"), "$env:CHORUS_DSN = 'synthetic'\n") + writeTestFile(t, filepath.Join(dir, "portal.yml"), "schema_version: 1\n") writeTestFile(t, filepath.Join(dir, "admin.yml"), "settings:\n application:\n host: 127.0.0.1\n port: 18090\n jwt:\n secret: synthetic-secret\n") path := filepath.Join(dir, "services.yml") writeTestFile(t, path, validConfig()) @@ -24,6 +25,9 @@ func TestLoadResolvesOnlyNonSensitiveRuntimeValues(t *testing.T) { if resolved.AdminUIAPIBase != "http://127.0.0.1:18090" || strings.Contains(strings.Join([]string{resolved.PortalAddress, resolved.AdminAddress, resolved.AdminUIAPIBase}, " "), "secret") { t.Fatalf("unexpected public projection: %#v", resolved) } + if resolved.PortalSettingsFile != filepath.Join(dir, "portal.yml") { + t.Fatalf("portal settings path = %q", resolved.PortalSettingsFile) + } } func TestLoadRejectsUnknownField(t *testing.T) { @@ -63,6 +67,7 @@ func TestLoadRejectsNonLoopbackAndDuplicateAddresses(t *testing.T) { func TestLoadRejectsMissingReferencedFile(t *testing.T) { dir := t.TempDir() writeTestFile(t, filepath.Join(dir, "portal.ps1"), "# synthetic\n") + writeTestFile(t, filepath.Join(dir, "portal.yml"), "schema_version: 1\n") path := filepath.Join(dir, "services.yml") writeTestFile(t, path, validConfig()) if _, err := Load(path); err == nil || !strings.Contains(err.Error(), "admin.settings_file file does not exist") { @@ -74,12 +79,13 @@ func validFixture(t *testing.T) string { t.Helper() dir := t.TempDir() writeTestFile(t, filepath.Join(dir, "portal.ps1"), "# synthetic\n") + writeTestFile(t, filepath.Join(dir, "portal.yml"), "schema_version: 1\n") writeTestFile(t, filepath.Join(dir, "admin.yml"), "settings:\n application:\n host: 127.0.0.1\n port: 18090\n") return dir } func validConfig() string { - return "schema_version: 1\nportal:\n host: 127.0.0.1\n port: 18080\n environment_file: portal.ps1\nadmin:\n settings_file: admin.yml\nadmin_ui:\n host: 127.0.0.1\n port: 19527\n" + return "schema_version: 1\nportal:\n host: 127.0.0.1\n port: 18080\n environment_file: portal.ps1\n settings_file: portal.yml\nadmin:\n settings_file: admin.yml\nadmin_ui:\n host: 127.0.0.1\n port: 19527\n" } func writeTestFile(t *testing.T, path, content string) { diff --git a/portal/config/settings.example.yml b/portal/config/settings.example.yml new file mode 100644 index 0000000..604f4d8 --- /dev/null +++ b/portal/config/settings.example.yml @@ -0,0 +1,12 @@ +schema_version: 1 + +provider: + http_timeout_seconds: 45 + max_response_bytes: 33554432 + allowed_ports: + - 80 + - 443 + +worker: + lease_seconds: 60 + poll_milliseconds: 250 diff --git a/portal/main.go b/portal/main.go index 1c3b990..7c7348a 100644 --- a/portal/main.go +++ b/portal/main.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "flag" "fmt" "log" "net/http" @@ -29,14 +30,20 @@ import ( ) func main() { - if err := run(); err != nil { + configPath := flag.String("config", "", "portal settings YAML file") + flag.Parse() + if flag.NArg() != 0 { + log.Printf("chorus portal stopped: unexpected positional arguments") + os.Exit(1) + } + if err := run(*configPath); err != nil { log.Printf("chorus portal stopped: %v", err) os.Exit(1) } } -func run() error { - cfg, err := config.Load() +func run(configPath string) error { + cfg, err := config.LoadFile(configPath) if err != nil { return err } diff --git a/scripts/chorus-all.ps1 b/scripts/chorus-all.ps1 index 4ca5c6f..b71bdc3 100644 --- a/scripts/chorus-all.ps1 +++ b/scripts/chorus-all.ps1 @@ -332,7 +332,7 @@ function Start-Portal { } $endpoint = ConvertTo-Endpoint $Resolved.portal_address Assert-PortAvailable $endpoint "portal" - & (Join-Path $PSScriptRoot "chorus-dev.bat") start $Resolved.portal_environment_file $Resolved.portal_address + & (Join-Path $PSScriptRoot "chorus-dev.bat") start $Resolved.portal_environment_file $Resolved.portal_address $Resolved.portal_settings_file if ($LASTEXITCODE -ne 0) { throw "Portal startup failed." } diff --git a/scripts/chorus-dev.ps1 b/scripts/chorus-dev.ps1 index a2dcbaf..a0164f6 100644 --- a/scripts/chorus-dev.ps1 +++ b/scripts/chorus-dev.ps1 @@ -8,7 +8,10 @@ param( [string]$EnvironmentFile, [Parameter(Position = 2)] - [string]$PortalAddress + [string]$PortalAddress, + + [Parameter(Position = 3)] + [string]$SettingsFile ) Set-StrictMode -Version Latest @@ -20,6 +23,7 @@ if ([string]::IsNullOrWhiteSpace($env:LOCALAPPDATA)) { $script:RepoRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..")) $script:EnvironmentFile = $EnvironmentFile +$script:SettingsFile = $SettingsFile $script:StateRoot = Join-Path $env:LOCALAPPDATA "Chorus\dev" $script:BinRoot = Join-Path $script:StateRoot "bin" $script:LogRoot = Join-Path $script:StateRoot "logs" @@ -153,6 +157,18 @@ function Get-PortalEndpoint { return Get-Endpoint $address "127.0.0.1:8080" "portal address" } +function Get-PortalSettingsFile { + if ([string]::IsNullOrWhiteSpace($script:SettingsFile)) { + $script:SettingsFile = Join-Path $script:RepoRoot "portal\config\settings.yml" + } + $resolved = Resolve-Path -LiteralPath $script:SettingsFile -ErrorAction SilentlyContinue + if ($null -eq $resolved -or $resolved.Path -notmatch '\.(yml|yaml)$') { + throw "Portal settings file is missing or invalid: $script:SettingsFile" + } + $script:SettingsFile = $resolved.Path + return $script:SettingsFile +} + function Test-TcpPort { param([string]$HostName, [int]$Port, [int]$TimeoutMilliseconds = 500) $client = [Net.Sockets.TcpClient]::new() @@ -259,7 +275,8 @@ function Start-Component { param( [ValidateSet("portal", "mock")][string]$Name, [string]$Package, - $Endpoint + $Endpoint, + [string[]]$Arguments = @() ) $existing = Get-ManagedProcess $Name if ($null -ne $existing) { @@ -272,7 +289,18 @@ function Start-Component { $stamp = Get-Date -Format "yyyyMMdd-HHmmss" $stdout = Join-Path $script:LogRoot "$Name-$stamp.out.log" $stderr = Join-Path $script:LogRoot "$Name-$stamp.err.log" - $process = Start-Process -FilePath $executable -WorkingDirectory $script:RepoRoot -RedirectStandardOutput $stdout -RedirectStandardError $stderr -PassThru -WindowStyle Hidden + $startArguments = @{ + FilePath = $executable + WorkingDirectory = $script:RepoRoot + RedirectStandardOutput = $stdout + RedirectStandardError = $stderr + PassThru = $true + WindowStyle = "Hidden" + } + if ($Arguments.Count -gt 0) { + $startArguments.ArgumentList = $Arguments + } + $process = Start-Process @startArguments $state = [ordered]@{ name = $Name pid = $process.Id @@ -361,7 +389,8 @@ try { Assert-DatabaseAvailable $portalEndpoint = Get-PortalEndpoint $env:CHORUS_LISTEN_ADDRESS = $portalEndpoint.Address - Start-Component "portal" "./portal" $portalEndpoint + $settingsArgument = '--config="' + (Get-PortalSettingsFile).Replace('"', '\"') + '"' + Start-Component "portal" "./portal" $portalEndpoint @($settingsArgument) } "start-mock" { Import-ChorusEnvironment @@ -375,7 +404,8 @@ try { Assert-DatabaseAvailable $portalEndpoint = Get-PortalEndpoint $env:CHORUS_LISTEN_ADDRESS = $portalEndpoint.Address - Start-Component "portal" "./portal" $portalEndpoint + $settingsArgument = '--config="' + (Get-PortalSettingsFile).Replace('"', '\"') + '"' + Start-Component "portal" "./portal" $portalEndpoint @($settingsArgument) } } } diff --git a/scripts/chorus-supervisor.ps1 b/scripts/chorus-supervisor.ps1 index 7d24f87..6b25e1a 100644 --- a/scripts/chorus-supervisor.ps1 +++ b/scripts/chorus-supervisor.ps1 @@ -212,7 +212,7 @@ function Run-Portal { } $env:CHORUS_LISTEN_ADDRESS = $resolved.portal_address $executable = Join-Path $script:BinRoot "chorus-portal.exe" - & $executable + & $executable --config $resolved.portal_settings_file exit $LASTEXITCODE }