Files
synapbus/internal/messaging/memory_config.go
T
Algis DumbrisandClaude Opus 4.7 bfb2551b45 feat(020): configurable dream parallelism + 3 bug fixes from kubic drain
Drain-on-demand: SYNAPBUS_DREAM_PARALLEL (default 1) and
`synapbus memory dream-run --parallel N` fan out N concurrent
dream-agent k8s Jobs per (owner, job_type) in one shot. Set
high (e.g. 8) to drain backlog quickly, then back to 1 for normal
hourly operation.

Schema:
- migration 030_dream_parallelism: adds slot INTEGER NOT NULL DEFAULT 0
  to memory_consolidation_jobs. Drops + recreates the partial unique
  in-flight index as (owner, job_type, slot) so slots 0..N-1 each hold
  one in-flight job independently.

Stores:
- JobsStore.CreateOnSlot + CreateNextAvailableSlot.
- ConsolidatorWorker.ForceRunN dispatches N parallel jobs through the
  existing launchOne path (extracted from ForceRun).
- core_rewrite coerces to N=1 regardless of the knob — per-(owner,
  agent) blob is wholesale-replace and concurrent rewrites would race.

Three bug fixes discovered while bringing the parallel path up on
kubic:

1. k8s Job names collided on rapid relaunch because runner.go used
   "synapbus-<agent>-<msg_id>", and dream dispatches have msg_id=0.
   Now appends a unique (timestamp%1e6, 4-byte random) suffix when
   msg_id is zero; historical "synapbus-<agent>-<id>" prefix preserved.

2. memory_list_unprocessed didn't actually exclude already-refined
   messages — the contract said it should, the implementation
   returned the same oldest-50 every cycle. The dream agent kept
   re-refining the same set: 221 refines links touched only 55
   unique dst messages, so progress flat-lined. Added the
   NOT IN (refines/duplicate_of/superseded_by) filter and a
   from_agent NOT LIKE 'dream:%' clause so the agent never refines
   its own reflections.

3. The k8sjob harness was constructed with nil Waiter in main.go,
   so every dream dispatch failed instantly with "k8sjob: no Waiter
   configured". Now builds a ClientsetWaiter from the in-cluster
   clientset.

Plus admin/server.go gets DreamRunN closure + DefaultDreamParallel
(sourced from MemoryConfig.DreamParallel). admin/socket.go
handleMemoryDreamRun accepts `parallel` arg and returns job_ids[].
CLI admin command grows --parallel N flag.

Live evidence from kubic (image v0.21.0-amd64):
  1 CLI call with --parallel 8 produced 8 job rows on slots 0..7,
  spawned 8 distinct k8s Jobs with unique suffixes, retired ~86
  unprocessed messages in <1 min (vs ~10/cycle for the buggy
  serial version pre-fix-2).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-12 19:53:25 +03:00

220 lines
7.2 KiB
Go

package messaging
import (
"os"
"strconv"
"time"
)
// MemoryConfig holds feature-flag and tunable knobs for proactive memory
// injection and the dream worker. All fields are populated from
// environment variables by ParseMemoryConfig; defaults match the values
// listed in the 020-proactive-memory-dream-worker spec.
type MemoryConfig struct {
// InjectionEnabled toggles the `relevant_context` packet on MCP tool
// responses. When false, the wrapper is a no-op and the field is
// omitted from responses entirely.
InjectionEnabled bool
// InjectionBudgetTokens is the soft cap for the merged packet
// (memories + core_memory), counted as `(chars+3)/4`.
InjectionBudgetTokens int
// InjectionMaxItems caps the number of memory items in a single packet.
InjectionMaxItems int
// InjectionMinScore is the relevance floor; items below this are
// dropped unless explicitly pinned.
InjectionMinScore float64
// CoreMemoryMaxBytes is the upper bound on a per-(owner, agent) core
// memory blob. Enforced in the Set path.
CoreMemoryMaxBytes int
// DreamEnabled gates the entire ConsolidatorWorker lifecycle and the
// six memory_* MCP tools.
DreamEnabled bool
// DreamInterval is the ticker period for the consolidator worker.
DreamInterval time.Duration
// DreamDeepCron is the cron expression for the nightly deep pass
// (e.g. core_rewrite). Standard 5-field cron; not parsed here.
DreamDeepCron string
// DreamMaxConcurrent caps the number of owners with in-flight jobs.
DreamMaxConcurrent int
// DreamParallel is the number of concurrent dream jobs the worker
// (and `synapbus memory dream-run`) will fan out per (owner, job_type)
// when triggered. 1 = historical single-job behaviour. Higher values
// let one watermark/manual trigger spawn N parallel reflection /
// link_gen / dedup_contradiction agents to drain a backlog. core_rewrite
// is always single-slot regardless of this knob.
DreamParallel int
// DreamWallclockBudget caps how long a single consolidation job is
// allowed to run before the worker terminates it with status=partial.
DreamWallclockBudget time.Duration
// DreamWatermark is the number of new memory messages required
// before a reflection job triggers automatically for an owner.
DreamWatermark int
// DreamAgent is the name of the agent invoked as the dream worker
// via harness.Harness.Execute.
DreamAgent string
// DreamRecentWindow bounds the lookback for the dream worker's
// per-owner input set (unprocessed-count queries, recency
// injection fallback, and memory_list_unprocessed). Messages older
// than `now - DreamRecentWindow` are invisible to the worker — the
// pool is effectively a rolling window of recent activity. Default
// 14 days. Env: SYNAPBUS_DREAM_RECENT_WINDOW. Accepts Go
// duration syntax plus the "Nd" days extension.
DreamRecentWindow time.Duration
// DreamDailyTokenLimitIn is the per-(owner, day) input-token
// circuit-breaker threshold. When the running sum of TokensIn
// across today's dream jobs exceeds this, the worker skips further
// dispatches for that owner until the date rolls over. Default 1M.
// Env: SYNAPBUS_DREAM_DAILY_TOKEN_LIMIT_IN.
DreamDailyTokenLimitIn int64
// DreamDailyTokenLimitOut is the per-(owner, day) output-token
// circuit-breaker threshold. Default 200k.
// Env: SYNAPBUS_DREAM_DAILY_TOKEN_LIMIT_OUT.
DreamDailyTokenLimitOut int64
// DreamDailyJobLimit is the per-(owner, day) ceiling on jobs
// started. Default 100. Env: SYNAPBUS_DREAM_DAILY_JOB_LIMIT.
DreamDailyJobLimit int
}
// DefaultMemoryConfig returns the defaults exactly as listed in the
// 020-proactive-memory-dream-worker spec.
func DefaultMemoryConfig() MemoryConfig {
return MemoryConfig{
InjectionEnabled: false,
InjectionBudgetTokens: 500,
InjectionMaxItems: 5,
InjectionMinScore: 0.25,
CoreMemoryMaxBytes: 2048,
DreamEnabled: false,
DreamInterval: 1 * time.Hour,
DreamDeepCron: "0 3 * * *",
DreamMaxConcurrent: 4,
DreamParallel: 1,
DreamWallclockBudget: 10 * time.Minute,
DreamWatermark: 20,
DreamAgent: "claude-code",
// 14 days of recent activity.
DreamRecentWindow: 336 * time.Hour,
DreamDailyTokenLimitIn: 1_000_000,
DreamDailyTokenLimitOut: 200_000,
DreamDailyJobLimit: 100,
}
}
// ParseMemoryConfig reads MemoryConfig fields from environment
// variables, falling back to the spec defaults for any unset or
// unparseable value.
func ParseMemoryConfig() MemoryConfig {
cfg := DefaultMemoryConfig()
if v := os.Getenv("SYNAPBUS_INJECTION_ENABLED"); v != "" {
if b, ok := parseBoolFlag(v); ok {
cfg.InjectionEnabled = b
}
}
if v := os.Getenv("SYNAPBUS_INJECTION_BUDGET_TOKENS"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
cfg.InjectionBudgetTokens = n
}
}
if v := os.Getenv("SYNAPBUS_INJECTION_MAX_ITEMS"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
cfg.InjectionMaxItems = n
}
}
if v := os.Getenv("SYNAPBUS_INJECTION_MIN_SCORE"); v != "" {
if f, err := strconv.ParseFloat(v, 64); err == nil && f >= 0 {
cfg.InjectionMinScore = f
}
}
if v := os.Getenv("SYNAPBUS_CORE_MEMORY_MAX_BYTES"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
cfg.CoreMemoryMaxBytes = n
}
}
if v := os.Getenv("SYNAPBUS_DREAM_ENABLED"); v != "" {
if b, ok := parseBoolFlag(v); ok {
cfg.DreamEnabled = b
}
}
if v := os.Getenv("SYNAPBUS_DREAM_INTERVAL"); v != "" {
if d, err := parseDurationWithDays(v); err == nil && d > 0 {
cfg.DreamInterval = d
}
}
if v := os.Getenv("SYNAPBUS_DREAM_DEEP_CRON"); v != "" {
cfg.DreamDeepCron = v
}
if v := os.Getenv("SYNAPBUS_DREAM_MAX_CONCURRENT"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
cfg.DreamMaxConcurrent = n
}
}
if v := os.Getenv("SYNAPBUS_DREAM_PARALLEL"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
cfg.DreamParallel = n
}
}
if v := os.Getenv("SYNAPBUS_DREAM_WALLCLOCK_BUDGET"); v != "" {
if d, err := parseDurationWithDays(v); err == nil && d > 0 {
cfg.DreamWallclockBudget = d
}
}
if v := os.Getenv("SYNAPBUS_DREAM_WATERMARK"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
cfg.DreamWatermark = n
}
}
if v := os.Getenv("SYNAPBUS_DREAM_AGENT"); v != "" {
cfg.DreamAgent = v
}
if v := os.Getenv("SYNAPBUS_DREAM_RECENT_WINDOW"); v != "" {
if d, err := parseDurationWithDays(v); err == nil && d > 0 {
cfg.DreamRecentWindow = d
}
}
if v := os.Getenv("SYNAPBUS_DREAM_DAILY_TOKEN_LIMIT_IN"); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
cfg.DreamDailyTokenLimitIn = n
}
}
if v := os.Getenv("SYNAPBUS_DREAM_DAILY_TOKEN_LIMIT_OUT"); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
cfg.DreamDailyTokenLimitOut = n
}
}
if v := os.Getenv("SYNAPBUS_DREAM_DAILY_JOB_LIMIT"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
cfg.DreamDailyJobLimit = n
}
}
return cfg
}
// parseBoolFlag accepts the common 0/1/true/false/yes/no spellings.
func parseBoolFlag(v string) (bool, bool) {
switch v {
case "1", "true", "TRUE", "True", "yes", "YES", "y", "on", "ON":
return true, true
case "0", "false", "FALSE", "False", "no", "NO", "n", "off", "OFF":
return false, true
}
return false, false
}