Backend (Go, in this commit):
- migration 029_memory_dream_usage: per (date, owner) counters for
tokens_in/out, jobs_started/succeeded/failed/circuit_broken
- DreamUsageStore + UsageGate (internal/messaging/dream_usage.go).
Gate inspects today's usage against new env knobs:
- SYNAPBUS_DREAM_RECENT_WINDOW (default 336h / 14d)
- SYNAPBUS_DREAM_DAILY_TOKEN_LIMIT_IN (default 1M)
- SYNAPBUS_DREAM_DAILY_TOKEN_LIMIT_OUT (default 200k)
- SYNAPBUS_DREAM_DAILY_JOB_LIMIT (default 100)
- Consolidator now bounds watermarks + core_rewrite eligibility by the
recency window. core_rewrite skipped for owners with no in-window
activity. ForceRun honors the breaker.
- Recency fallback in BuildContextPacket + memory_list_unprocessed now
accept RecentWindowDays so injection and dream queries see the same
14d slice.
- Prometheus metrics registered (internal/metrics/metrics.go):
synapbus_dream_jobs_total{owner,job_type,status},
synapbus_dream_tokens_total{owner,direction},
synapbus_dream_job_duration_seconds{owner,job_type},
synapbus_dream_circuit_broken_total{owner,reason},
synapbus_injection_packets_total{tool},
synapbus_injection_memories_per_packet{tool},
synapbus_injection_packet_chars{tool},
synapbus_injection_skipped_total{tool,reason}.
- deploy/kubic/deployment.yaml: liveness/readiness timeoutSeconds: 1→5
(root-causes the "connection refused" mcpproxy errors at 13:02 today —
/readyz occasionally exceeded 1s under dream-worker tick load, so the
pod fell out of the Service endpoints intermittently).
Dream-claude agent (Python, in /dream-agent/):
- dream_runner.py uses claude-agent-sdk 0.1.48 to drive Claude Code
against SynapBus's MCP server. MCP transport carries
Authorization: Bearer <api_key> AND X-Synapbus-Dispatch-Token from env
via the SDK's McpHttpServerConfig.headers field — confirmed supported.
- Tools restricted via allowed_tools to mcp__synapbus__memory_*.
- Final JSON envelope reports tokens_in/out so harness.Usage stays
populated and the circuit breaker can count consumption.
- Dockerfile builds linux/amd64 at 189 MB, mirroring searcher's
agents/universal recipe.
- k8s-job-template.yaml: backoffLimit 0, ttl 600s, 512Mi/1CPU,
Anthropic credentials via secret-ref.
Grafana dashboard (deploy/kubic/grafana/):
- dream-dashboard.json — 14 panels across 5 rows (dream activity,
token usage vs limit, circuit breaker, injection layer, MCP
transport health), all templated to ${DS_PROMETHEUS}.
- import.sh: resolves the cluster's Prometheus DS uid and POSTs the
dashboard via Grafana API.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
199 lines
6.8 KiB
Go
199 lines
6.8 KiB
Go
// Dream-worker daily usage tracker + circuit breaker (feature 020
|
|
// follow-up). Wraps the `memory_dream_usage` table from migration 029.
|
|
//
|
|
// Each row aggregates one (UTC date, owner_id) bucket. The
|
|
// ConsolidatorWorker:
|
|
//
|
|
// 1. Calls UsageGate.Allow() before Create+Issue+Execute. If today's
|
|
// counters exceed any configured threshold the gate returns
|
|
// allowed=false with a reason code; the worker then records a
|
|
// `circuit_broken` job row and skips dispatch.
|
|
// 2. On successful Execute completion, calls RecordCompletion with
|
|
// the harness.ExecResult.Usage tokens so tomorrow's gate decisions
|
|
// incorporate today's spend.
|
|
//
|
|
// The circuit "resets" naturally — Today() pivots on UTC date so the
|
|
// first call after midnight UTC reads a fresh row with zero counters.
|
|
package messaging
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
// DreamDailyUsage is one row of `memory_dream_usage`.
|
|
type DreamDailyUsage struct {
|
|
Date string `json:"date"`
|
|
OwnerID string `json:"owner_id"`
|
|
TokensIn int64 `json:"tokens_in"`
|
|
TokensOut int64 `json:"tokens_out"`
|
|
JobsStarted int `json:"jobs_started"`
|
|
JobsSucceeded int `json:"jobs_succeeded"`
|
|
JobsFailed int `json:"jobs_failed"`
|
|
JobsCircuitBroken int `json:"jobs_circuit_broken"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
|
|
// DreamUsageStore wraps the `memory_dream_usage` table.
|
|
type DreamUsageStore struct {
|
|
db *sql.DB
|
|
// now is overridable in tests so callers can pin the UTC date used
|
|
// when bucketing.
|
|
now func() time.Time
|
|
}
|
|
|
|
// NewDreamUsageStore returns a store rooted at db.
|
|
func NewDreamUsageStore(db *sql.DB) *DreamUsageStore {
|
|
return &DreamUsageStore{db: db, now: time.Now}
|
|
}
|
|
|
|
// utcDate returns the YYYY-MM-DD key for the store's clock.
|
|
func (s *DreamUsageStore) utcDate() string {
|
|
return s.now().UTC().Format("2006-01-02")
|
|
}
|
|
|
|
// upsert is the common path for all increments — UPSERT (date, owner_id).
|
|
// Counters are added with COALESCE so deltas accumulate cleanly.
|
|
func (s *DreamUsageStore) upsert(ctx context.Context, ownerID string,
|
|
dIn, dOut int64, dStart, dSuc, dFail, dCB int,
|
|
) error {
|
|
if s == nil || s.db == nil {
|
|
return nil
|
|
}
|
|
if ownerID == "" {
|
|
return fmt.Errorf("dream usage: empty owner_id")
|
|
}
|
|
date := s.utcDate()
|
|
_, err := s.db.ExecContext(ctx, `
|
|
INSERT INTO memory_dream_usage
|
|
(date, owner_id, tokens_in, tokens_out,
|
|
jobs_started, jobs_succeeded, jobs_failed, jobs_circuit_broken, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
|
ON CONFLICT(date, owner_id) DO UPDATE SET
|
|
tokens_in = tokens_in + excluded.tokens_in,
|
|
tokens_out = tokens_out + excluded.tokens_out,
|
|
jobs_started = jobs_started + excluded.jobs_started,
|
|
jobs_succeeded = jobs_succeeded + excluded.jobs_succeeded,
|
|
jobs_failed = jobs_failed + excluded.jobs_failed,
|
|
jobs_circuit_broken = jobs_circuit_broken + excluded.jobs_circuit_broken,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
`, date, ownerID, dIn, dOut, dStart, dSuc, dFail, dCB)
|
|
if err != nil {
|
|
return fmt.Errorf("dream usage: upsert: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RecordStart increments today's jobs_started for ownerID.
|
|
func (s *DreamUsageStore) RecordStart(ctx context.Context, ownerID string) error {
|
|
return s.upsert(ctx, ownerID, 0, 0, 1, 0, 0, 0)
|
|
}
|
|
|
|
// RecordCompletion increments tokens and the appropriate per-status
|
|
// counter. status ∈ {succeeded, failed, circuit_broken}. Unknown
|
|
// statuses are counted as failed so the gate errs on the safe side.
|
|
func (s *DreamUsageStore) RecordCompletion(ctx context.Context, ownerID string, tokensIn, tokensOut int64, status string) error {
|
|
var dSuc, dFail, dCB int
|
|
switch status {
|
|
case JobStatusSucceeded:
|
|
dSuc = 1
|
|
case JobStatusCircuitBroken:
|
|
dCB = 1
|
|
case JobStatusPartial:
|
|
// Partial counts as succeeded for the circuit-breaker — it ran.
|
|
dSuc = 1
|
|
default:
|
|
dFail = 1
|
|
}
|
|
return s.upsert(ctx, ownerID, tokensIn, tokensOut, 0, dSuc, dFail, dCB)
|
|
}
|
|
|
|
// Today returns today's counters for ownerID. Missing row → zero-value.
|
|
func (s *DreamUsageStore) Today(ctx context.Context, ownerID string) (DreamDailyUsage, error) {
|
|
if s == nil || s.db == nil {
|
|
return DreamDailyUsage{}, nil
|
|
}
|
|
if ownerID == "" {
|
|
return DreamDailyUsage{}, fmt.Errorf("dream usage: empty owner_id")
|
|
}
|
|
date := s.utcDate()
|
|
u := DreamDailyUsage{Date: date, OwnerID: ownerID}
|
|
err := s.db.QueryRowContext(ctx, `
|
|
SELECT tokens_in, tokens_out, jobs_started, jobs_succeeded,
|
|
jobs_failed, jobs_circuit_broken, updated_at
|
|
FROM memory_dream_usage
|
|
WHERE date = ? AND owner_id = ?
|
|
`, date, ownerID).Scan(
|
|
&u.TokensIn, &u.TokensOut, &u.JobsStarted, &u.JobsSucceeded,
|
|
&u.JobsFailed, &u.JobsCircuitBroken, &u.UpdatedAt,
|
|
)
|
|
if err == sql.ErrNoRows {
|
|
return u, nil
|
|
}
|
|
if err != nil {
|
|
return u, fmt.Errorf("dream usage: today: %w", err)
|
|
}
|
|
return u, nil
|
|
}
|
|
|
|
// Cleanup deletes usage rows older than olderThanDays. Default callers
|
|
// can pass 30 to keep a month of history for diagnostics.
|
|
func (s *DreamUsageStore) Cleanup(ctx context.Context, olderThanDays int) (int64, error) {
|
|
if s == nil || s.db == nil {
|
|
return 0, nil
|
|
}
|
|
if olderThanDays <= 0 {
|
|
return 0, nil
|
|
}
|
|
cutoff := s.now().UTC().AddDate(0, 0, -olderThanDays).Format("2006-01-02")
|
|
res, err := s.db.ExecContext(ctx,
|
|
`DELETE FROM memory_dream_usage WHERE date < ?`, cutoff,
|
|
)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("dream usage: cleanup: %w", err)
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
return n, nil
|
|
}
|
|
|
|
// UsageGate is the circuit breaker the ConsolidatorWorker consults
|
|
// before dispatching a job. It reads today's counters via DreamUsageStore
|
|
// and compares them against the per-day limits in MemoryConfig.
|
|
type UsageGate struct {
|
|
cfg MemoryConfig
|
|
store *DreamUsageStore
|
|
}
|
|
|
|
// NewUsageGate ties a MemoryConfig to a DreamUsageStore.
|
|
func NewUsageGate(cfg MemoryConfig, store *DreamUsageStore) *UsageGate {
|
|
return &UsageGate{cfg: cfg, store: store}
|
|
}
|
|
|
|
// Allow returns (allowed, reasonCode, err). When allowed is false the
|
|
// caller should record a `circuit_broken` job and skip dispatch.
|
|
// reasonCode is one of {tokens_in_exceeded, tokens_out_exceeded,
|
|
// jobs_exceeded, ""}. err is non-nil only for database failures —
|
|
// callers may treat err != nil as "fail open" to avoid wedging the
|
|
// worker on a transient store glitch.
|
|
func (g *UsageGate) Allow(ctx context.Context, ownerID string) (bool, string, error) {
|
|
if g == nil || g.store == nil {
|
|
return true, "", nil
|
|
}
|
|
u, err := g.store.Today(ctx, ownerID)
|
|
if err != nil {
|
|
return true, "", err
|
|
}
|
|
if g.cfg.DreamDailyTokenLimitIn > 0 && u.TokensIn >= g.cfg.DreamDailyTokenLimitIn {
|
|
return false, "tokens_in_exceeded", nil
|
|
}
|
|
if g.cfg.DreamDailyTokenLimitOut > 0 && u.TokensOut >= g.cfg.DreamDailyTokenLimitOut {
|
|
return false, "tokens_out_exceeded", nil
|
|
}
|
|
if g.cfg.DreamDailyJobLimit > 0 && u.JobsStarted >= g.cfg.DreamDailyJobLimit {
|
|
return false, "jobs_exceeded", nil
|
|
}
|
|
return true, "", nil
|
|
}
|