feat(017): MuSiQue benchmark harness — marketplace stub, mixed-tier agents, Pareto scoring, HTML report
MVP implementation of the MuSiQue multi-agent benchmark (spec 017): - benchmark/setup.py: downloads musique_v1.0.zip from the canonical Google Drive source (mirrors upstream download_data.sh). Idempotent. - benchmark/curate.py: deterministic selection of 3 4-hop questions from the dev set sharing a US pivot entity; writes benchmark/trio.jsonl. - benchmark/marketplace.py: in-process 016-marketplace stub with post_auction / bid / award / mark_done / query_reputation and a domain-scoped reputation ledger. Designed for mechanical swap to real SynapBus MCP tools. - benchmark/agents.py: HaikuAgent + SonnetAgent, using the official anthropic SDK (no Claude Agent SDK, no subprocesses). Models pinned to claude-haiku-4-5-20251001 and claude-sonnet-4-6. - benchmark/baseline.py: single Sonnet call with all 20 distractors plus chain-of-thought. - benchmark/score.py: SQuAD-style normalized F1 + Pareto verdict (strictly northwest = PASS). - benchmark/run.py: main entry. --mode single-shot, --question, --dry-run. - benchmark/report.py: self-contained HTML with inline SVG scatter plot. - benchmark/trio.jsonl: curated reproducible trio (all three converge on "Treaty of Paris" US territory cession). Verified with benchmark/run.py --dry-run end-to-end; all 8 files py_compile clean. Real-token execution is deferred to the user's main session. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Algis Dumbris
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
e77fd7afdf
commit
02b8548eac
@@ -41,6 +41,12 @@ Thumbs.db
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Benchmark (017) — large datasets, per-run outputs, local venvs
|
||||
benchmark/data/
|
||||
benchmark/results/
|
||||
.venv-bench/
|
||||
.venv/
|
||||
|
||||
# Debug
|
||||
__debug_bin*
|
||||
.claude/worktrees/
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Agent pool for the MuSiQue benchmark.
|
||||
|
||||
Two agents:
|
||||
- haiku-agent (claude-haiku-4-5-20251001)
|
||||
- sonnet-agent (claude-sonnet-4-6)
|
||||
|
||||
Each agent exposes:
|
||||
- name, model, skill_card
|
||||
- bid(task) -> {estimated_tokens, confidence, approach}
|
||||
- execute(task, paragraphs) -> {answer, actual_tokens}
|
||||
|
||||
Design notes:
|
||||
- We use the official ``anthropic`` Python SDK directly (NOT the
|
||||
Claude Agent SDK). Simpler, no subprocesses, reliable token accounting.
|
||||
- ``bid()`` is pure Python — it is a cheap heuristic so the marketplace
|
||||
has something to pick from. Real 016 agents would emit a structured
|
||||
reply. For MVP, heuristic bids are sufficient to exercise the auction
|
||||
primitive.
|
||||
- ``execute()`` is the only thing that actually burns tokens.
|
||||
- ``--dry-run`` in run.py never calls execute(); it uses stub responses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import anthropic # type: ignore
|
||||
except ImportError: # pragma: no cover
|
||||
anthropic = None # type: ignore
|
||||
|
||||
|
||||
HAIKU_MODEL = "claude-haiku-4-5-20251001"
|
||||
SONNET_MODEL = "claude-sonnet-4-6"
|
||||
|
||||
|
||||
HAIKU_SKILL_CARD = """\
|
||||
# haiku-agent
|
||||
|
||||
A fast, cheap agent best for single-hop fact lookups and short
|
||||
extractive answers. Accepts multi-paragraph context but may miss
|
||||
subtle bridging entities on 4-hop questions. Very low cost per call.
|
||||
|
||||
Domains: factual-lookup, extraction, summarization
|
||||
"""
|
||||
|
||||
SONNET_SKILL_CARD = """\
|
||||
# sonnet-agent
|
||||
|
||||
A deliberate mid-tier agent well-suited to multi-hop reasoning with
|
||||
explicit chain-of-thought. Handles 4-hop MuSiQue questions with
|
||||
decomposition when the context fits in one prompt. Higher cost per call
|
||||
than Haiku but meaningfully better F1 on bridging questions.
|
||||
|
||||
Domains: multi-hop-qa, decomposition, reasoning
|
||||
"""
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """\
|
||||
You are a careful question-answering agent working on a MuSiQue
|
||||
multi-hop benchmark. You are given a question and a set of numbered
|
||||
paragraphs. Only a few of the paragraphs are relevant; the rest are
|
||||
distractors.
|
||||
|
||||
Think step by step and cite the paragraphs you used. Then output a
|
||||
final line starting with exactly:
|
||||
|
||||
ANSWER: <your short final answer>
|
||||
|
||||
Your final answer must be a short entity or phrase — not a sentence.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class BidResult:
|
||||
estimated_tokens: int
|
||||
confidence: float
|
||||
approach: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"estimated_tokens": self.estimated_tokens,
|
||||
"confidence": self.confidence,
|
||||
"approach": self.approach,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecuteResult:
|
||||
answer: str
|
||||
actual_tokens: int
|
||||
raw_text: str = ""
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"answer": self.answer,
|
||||
"actual_tokens": self.actual_tokens,
|
||||
}
|
||||
|
||||
|
||||
class Agent:
|
||||
name: str
|
||||
model: str
|
||||
skill_card: str
|
||||
|
||||
def __init__(self, name: str, model: str, skill_card: str) -> None:
|
||||
self.name = name
|
||||
self.model = model
|
||||
self.skill_card = skill_card
|
||||
|
||||
# ---- bidding -----------------------------------------------------------
|
||||
|
||||
def bid(self, task: dict[str, Any]) -> BidResult:
|
||||
raise NotImplementedError
|
||||
|
||||
# ---- execution ---------------------------------------------------------
|
||||
|
||||
def execute(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
paragraphs: list[str],
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
max_budget_tokens: int = 100_000,
|
||||
) -> ExecuteResult:
|
||||
question = task["question"]
|
||||
prompt = self._build_prompt(question, paragraphs)
|
||||
|
||||
if dry_run:
|
||||
stub = (
|
||||
"Thinking step by step... [dry-run stub]\n"
|
||||
f"ANSWER: [stub answer from {self.name}]"
|
||||
)
|
||||
# Rough estimate: 1 token ~= 4 characters.
|
||||
est = max(256, len(prompt) // 4 + 64)
|
||||
return ExecuteResult(
|
||||
answer=self._extract_answer(stub),
|
||||
actual_tokens=est,
|
||||
raw_text=stub,
|
||||
)
|
||||
|
||||
if anthropic is None:
|
||||
raise RuntimeError(
|
||||
"anthropic SDK not installed — pip install anthropic"
|
||||
)
|
||||
api_key = os.environ.get("ANTHROPIC_API_KEY")
|
||||
if not api_key:
|
||||
raise RuntimeError(
|
||||
"ANTHROPIC_API_KEY not set. Use --dry-run to stub it out."
|
||||
)
|
||||
|
||||
client = anthropic.Anthropic(api_key=api_key)
|
||||
# Cap max_tokens to min(1024, budget/2) so the worst case is tame.
|
||||
max_tokens = min(1024, max(128, max_budget_tokens // 2))
|
||||
msg = client.messages.create(
|
||||
model=self.model,
|
||||
max_tokens=max_tokens,
|
||||
system=SYSTEM_PROMPT,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
)
|
||||
text_parts: list[str] = []
|
||||
for block in msg.content:
|
||||
t = getattr(block, "text", None)
|
||||
if t:
|
||||
text_parts.append(t)
|
||||
text = "\n".join(text_parts).strip()
|
||||
|
||||
usage = getattr(msg, "usage", None)
|
||||
actual = 0
|
||||
if usage is not None:
|
||||
actual = (
|
||||
getattr(usage, "input_tokens", 0)
|
||||
+ getattr(usage, "output_tokens", 0)
|
||||
)
|
||||
return ExecuteResult(
|
||||
answer=self._extract_answer(text),
|
||||
actual_tokens=int(actual),
|
||||
raw_text=text,
|
||||
)
|
||||
|
||||
# ---- helpers -----------------------------------------------------------
|
||||
|
||||
def _build_prompt(
|
||||
self, question: str, paragraphs: list[str]
|
||||
) -> str:
|
||||
body = ["Paragraphs:"]
|
||||
for i, p in enumerate(paragraphs, start=1):
|
||||
body.append(f"[{i}] {p}")
|
||||
body.append("")
|
||||
body.append(f"Question: {question}")
|
||||
body.append("")
|
||||
body.append("Think step by step, then output your final ANSWER: line.")
|
||||
return "\n".join(body)
|
||||
|
||||
def _extract_answer(self, text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
for line in reversed(text.splitlines()):
|
||||
line = line.strip()
|
||||
if line.upper().startswith("ANSWER:"):
|
||||
return line.split(":", 1)[1].strip()
|
||||
# Fallback: last non-empty line.
|
||||
for line in reversed(text.splitlines()):
|
||||
line = line.strip()
|
||||
if line:
|
||||
return line
|
||||
return ""
|
||||
|
||||
|
||||
class HaikuAgent(Agent):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
name="haiku-agent",
|
||||
model=HAIKU_MODEL,
|
||||
skill_card=HAIKU_SKILL_CARD,
|
||||
)
|
||||
|
||||
def bid(self, task: dict[str, Any]) -> BidResult:
|
||||
# Cheap, low confidence on multi-hop bridging.
|
||||
return BidResult(
|
||||
estimated_tokens=4_000,
|
||||
confidence=0.45,
|
||||
approach=(
|
||||
"Extract candidate entities from the paragraphs and "
|
||||
"answer directly; may miss 4-hop bridges."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SonnetAgent(Agent):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
name="sonnet-agent",
|
||||
model=SONNET_MODEL,
|
||||
skill_card=SONNET_SKILL_CARD,
|
||||
)
|
||||
|
||||
def bid(self, task: dict[str, Any]) -> BidResult:
|
||||
# More expensive, higher confidence on multi-hop.
|
||||
return BidResult(
|
||||
estimated_tokens=12_000,
|
||||
confidence=0.80,
|
||||
approach=(
|
||||
"Decompose the question into sub-questions, resolve each "
|
||||
"sub-answer against the paragraphs, then compose the final "
|
||||
"bridged answer."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def default_pool() -> list[Agent]:
|
||||
return [HaikuAgent(), SonnetAgent()]
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Single-agent baseline: one Anthropic API call to claude-sonnet-4-6 with
|
||||
the question and all 20 distractor paragraphs plus chain-of-thought
|
||||
instructions. No decomposition, no marketplace, no tools.
|
||||
|
||||
Returns {"answer": str, "tokens": int, "raw_text": str}.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import anthropic # type: ignore
|
||||
except ImportError: # pragma: no cover
|
||||
anthropic = None # type: ignore
|
||||
|
||||
|
||||
BASELINE_MODEL = "claude-sonnet-4-6"
|
||||
|
||||
BASELINE_SYSTEM = """\
|
||||
You are a careful multi-hop QA system. Given a question and a set of
|
||||
numbered paragraphs (some irrelevant distractors), think step by step
|
||||
and answer.
|
||||
|
||||
Output your reasoning first, then on a final line:
|
||||
|
||||
ANSWER: <short final answer>
|
||||
"""
|
||||
|
||||
|
||||
def _build_prompt(question: str, paragraphs: list[str]) -> str:
|
||||
parts = ["Paragraphs:"]
|
||||
for i, p in enumerate(paragraphs, start=1):
|
||||
parts.append(f"[{i}] {p}")
|
||||
parts.append("")
|
||||
parts.append(f"Question: {question}")
|
||||
parts.append("")
|
||||
parts.append(
|
||||
"Work through the reasoning step by step, then give your "
|
||||
"final ANSWER: line."
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _extract_answer(text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
for line in reversed(text.splitlines()):
|
||||
line = line.strip()
|
||||
if line.upper().startswith("ANSWER:"):
|
||||
return line.split(":", 1)[1].strip()
|
||||
for line in reversed(text.splitlines()):
|
||||
line = line.strip()
|
||||
if line:
|
||||
return line
|
||||
return ""
|
||||
|
||||
|
||||
def run_baseline(
|
||||
question: str,
|
||||
paragraphs: list[str],
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
max_output_tokens: int = 1024,
|
||||
) -> dict[str, Any]:
|
||||
prompt = _build_prompt(question, paragraphs)
|
||||
|
||||
if dry_run:
|
||||
stub = (
|
||||
"Step 1: scanning paragraphs... [dry-run stub]\n"
|
||||
"Step 2: picking the most likely entity...\n"
|
||||
"ANSWER: [stub baseline answer]"
|
||||
)
|
||||
est = max(512, len(prompt) // 4 + 128)
|
||||
return {
|
||||
"answer": _extract_answer(stub),
|
||||
"tokens": est,
|
||||
"raw_text": stub,
|
||||
"model": BASELINE_MODEL,
|
||||
}
|
||||
|
||||
if anthropic is None:
|
||||
raise RuntimeError("anthropic SDK not installed")
|
||||
api_key = os.environ.get("ANTHROPIC_API_KEY")
|
||||
if not api_key:
|
||||
raise RuntimeError("ANTHROPIC_API_KEY not set")
|
||||
|
||||
client = anthropic.Anthropic(api_key=api_key)
|
||||
msg = client.messages.create(
|
||||
model=BASELINE_MODEL,
|
||||
max_tokens=max_output_tokens,
|
||||
system=BASELINE_SYSTEM,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
)
|
||||
text_parts: list[str] = []
|
||||
for block in msg.content:
|
||||
t = getattr(block, "text", None)
|
||||
if t:
|
||||
text_parts.append(t)
|
||||
text = "\n".join(text_parts).strip()
|
||||
usage = getattr(msg, "usage", None)
|
||||
tokens = 0
|
||||
if usage is not None:
|
||||
tokens = (
|
||||
getattr(usage, "input_tokens", 0)
|
||||
+ getattr(usage, "output_tokens", 0)
|
||||
)
|
||||
return {
|
||||
"answer": _extract_answer(text),
|
||||
"tokens": int(tokens),
|
||||
"raw_text": text,
|
||||
"model": BASELINE_MODEL,
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Curate a deterministic trio of MuSiQue 4-hop questions that share a
|
||||
pivot entity. For MVP we pivot on the United States.
|
||||
|
||||
Input: benchmark/data/musique_ans_v1.0_dev.jsonl
|
||||
Output: benchmark/trio.jsonl
|
||||
|
||||
Each output record:
|
||||
{
|
||||
"id": str,
|
||||
"question": str,
|
||||
"answer": str,
|
||||
"decomposition": [{"question": str, "answer": str}, ...],
|
||||
"paragraphs": [str, ...] # up to 20 distractor snippets
|
||||
}
|
||||
|
||||
MuSiQue dev records typically look like::
|
||||
|
||||
{
|
||||
"id": "4hop1__...",
|
||||
"question": "...",
|
||||
"question_decomposition": [
|
||||
{"id": N, "question": "...", "answer": "...",
|
||||
"paragraph_support_idx": int},
|
||||
...
|
||||
],
|
||||
"answer": "...",
|
||||
"answer_aliases": [...],
|
||||
"paragraphs": [
|
||||
{"idx": int, "title": "...", "paragraph_text": "...",
|
||||
"is_supporting": bool},
|
||||
...
|
||||
]
|
||||
}
|
||||
|
||||
The curation rule is deterministic (fixed input ordering; first 3 matches).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
DATA_FILE = Path(__file__).resolve().parent / "data" / "musique_ans_v1.0_dev.jsonl"
|
||||
OUT_FILE = Path(__file__).resolve().parent / "trio.jsonl"
|
||||
|
||||
PIVOT_TOKENS = ("united states", "u.s.", " us ", "america", "american")
|
||||
N_QUESTIONS = 3
|
||||
MAX_PARAGRAPHS = 20
|
||||
|
||||
|
||||
def _normalized(s: str) -> str:
|
||||
return f" {s.lower()} "
|
||||
|
||||
|
||||
def _mentions_pivot(record: dict) -> bool:
|
||||
blob_parts = [record.get("question", ""), record.get("answer", "")]
|
||||
for sub in record.get("question_decomposition", []) or []:
|
||||
blob_parts.append(sub.get("question", ""))
|
||||
blob_parts.append(sub.get("answer", ""))
|
||||
blob = _normalized(" ".join(str(x) for x in blob_parts if x))
|
||||
return any(tok in blob for tok in PIVOT_TOKENS)
|
||||
|
||||
|
||||
def _is_4hop(record: dict) -> bool:
|
||||
rid = record.get("id", "")
|
||||
if isinstance(rid, str) and rid.startswith("4hop"):
|
||||
return True
|
||||
# Fall back: count decomposition hops.
|
||||
decomp = record.get("question_decomposition") or []
|
||||
return len(decomp) == 4
|
||||
|
||||
|
||||
def _trim_paragraphs(record: dict, limit: int) -> list[str]:
|
||||
out: list[str] = []
|
||||
for p in record.get("paragraphs", []) or []:
|
||||
title = (p.get("title") or "").strip()
|
||||
text = (p.get("paragraph_text") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
snippet = f"[{title}] {text}" if title else text
|
||||
out.append(snippet)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _simplify_decomp(record: dict) -> list[dict]:
|
||||
out = []
|
||||
for sub in record.get("question_decomposition", []) or []:
|
||||
out.append(
|
||||
{
|
||||
"question": sub.get("question", ""),
|
||||
"answer": sub.get("answer", ""),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def curate() -> int:
|
||||
if not DATA_FILE.exists():
|
||||
print(
|
||||
f"[curate] ERROR: {DATA_FILE} not found. Run setup.py first.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
selected: list[dict] = []
|
||||
total_scanned = 0
|
||||
total_4hop = 0
|
||||
total_pivot = 0
|
||||
|
||||
with open(DATA_FILE, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
total_scanned += 1
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not _is_4hop(rec):
|
||||
continue
|
||||
total_4hop += 1
|
||||
if not _mentions_pivot(rec):
|
||||
continue
|
||||
total_pivot += 1
|
||||
|
||||
trio_record = {
|
||||
"id": rec.get("id", f"q{len(selected)+1}"),
|
||||
"question": rec.get("question", ""),
|
||||
"answer": rec.get("answer", ""),
|
||||
"answer_aliases": rec.get("answer_aliases", []),
|
||||
"decomposition": _simplify_decomp(rec),
|
||||
"paragraphs": _trim_paragraphs(rec, MAX_PARAGRAPHS),
|
||||
}
|
||||
selected.append(trio_record)
|
||||
if len(selected) >= N_QUESTIONS:
|
||||
break
|
||||
|
||||
print(
|
||||
f"[curate] scanned={total_scanned} 4hop={total_4hop} "
|
||||
f"pivot-matches={total_pivot} kept={len(selected)}"
|
||||
)
|
||||
|
||||
if len(selected) < N_QUESTIONS:
|
||||
print(
|
||||
f"[curate] ERROR: wanted {N_QUESTIONS} questions, "
|
||||
f"found {len(selected)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 3
|
||||
|
||||
OUT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(OUT_FILE, "w", encoding="utf-8") as f:
|
||||
for i, rec in enumerate(selected, start=1):
|
||||
# Attach a stable short id q1/q2/q3 in addition to MuSiQue's id.
|
||||
rec["short_id"] = f"q{i}"
|
||||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
|
||||
print(f"[curate] wrote {OUT_FILE} ({len(selected)} records)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(curate())
|
||||
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
In-process stub of the 016-agent-marketplace primitives.
|
||||
|
||||
*** IMPORTANT ***
|
||||
This module is an in-process stand-in for the SynapBus-hosted 016
|
||||
marketplace. The follow-up deliverable after this MVP is to replace the
|
||||
bodies of these functions with calls to the real SynapBus MCP tools
|
||||
(``post_auction``, ``bid``, ``award``, ``mark_done``, and
|
||||
``query_reputation``) once 016 lands. The public API here deliberately
|
||||
mirrors those tool names so the swap is mechanical.
|
||||
|
||||
Scope for MVP:
|
||||
- In-memory auctions, bids, awards, and done records
|
||||
- Domain-scoped reputation ledger stored in a dict
|
||||
- No persistence, no concurrency — single process, single thread
|
||||
- No schema enforcement beyond a couple of shape checks
|
||||
|
||||
The harness (``run.py``) holds a single ``Marketplace`` instance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class Auction:
|
||||
auction_id: str
|
||||
task: dict[str, Any]
|
||||
domain: str
|
||||
max_budget_tokens: int
|
||||
posted_at: float
|
||||
bids: list[dict[str, Any]] = field(default_factory=list)
|
||||
awarded_to: str | None = None
|
||||
result: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReputationEntry:
|
||||
agent: str
|
||||
domain: str
|
||||
runs: int = 0
|
||||
correct: int = 0
|
||||
tokens_spent: int = 0
|
||||
|
||||
def score(self) -> float:
|
||||
if self.runs == 0:
|
||||
return 0.5 # prior
|
||||
quality = self.correct / self.runs
|
||||
avg_tokens = self.tokens_spent / self.runs
|
||||
# Arbitrary: quality dominates, tokens slightly penalize.
|
||||
return quality - min(avg_tokens / 200_000.0, 0.3)
|
||||
|
||||
|
||||
class Marketplace:
|
||||
"""In-process marketplace stub."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._auctions: dict[str, Auction] = {}
|
||||
self._reputation: dict[tuple[str, str], ReputationEntry] = {}
|
||||
self._counter = itertools.count(1)
|
||||
|
||||
# ---- auction lifecycle -------------------------------------------------
|
||||
|
||||
def post_auction(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
domain: str,
|
||||
max_budget_tokens: int,
|
||||
) -> str:
|
||||
auction_id = f"auction-{next(self._counter)}"
|
||||
self._auctions[auction_id] = Auction(
|
||||
auction_id=auction_id,
|
||||
task=dict(task),
|
||||
domain=domain,
|
||||
max_budget_tokens=max_budget_tokens,
|
||||
posted_at=time.time(),
|
||||
)
|
||||
return auction_id
|
||||
|
||||
def bid(
|
||||
self,
|
||||
auction_id: str,
|
||||
agent: str,
|
||||
estimated_tokens: int,
|
||||
confidence: float,
|
||||
approach: str,
|
||||
) -> None:
|
||||
auction = self._auctions[auction_id]
|
||||
if auction.awarded_to is not None:
|
||||
raise RuntimeError(f"auction {auction_id} already awarded")
|
||||
auction.bids.append(
|
||||
{
|
||||
"agent": agent,
|
||||
"estimated_tokens": int(estimated_tokens),
|
||||
"confidence": float(confidence),
|
||||
"approach": approach,
|
||||
"submitted_at": time.time(),
|
||||
}
|
||||
)
|
||||
|
||||
def list_bids(self, auction_id: str) -> list[dict[str, Any]]:
|
||||
return list(self._auctions[auction_id].bids)
|
||||
|
||||
def score_bid(self, auction_id: str, bid: dict[str, Any]) -> float:
|
||||
"""
|
||||
Lower is better (we're minimizing tokens per unit confidence),
|
||||
but we add a reputation adjustment that rewards agents with a
|
||||
track record in this domain.
|
||||
"""
|
||||
auction = self._auctions[auction_id]
|
||||
rep = self._reputation.get((bid["agent"], auction.domain))
|
||||
rep_score = rep.score() if rep else 0.5
|
||||
conf = max(bid["confidence"], 1e-3)
|
||||
# Cost per confidence, lightly discounted by reputation.
|
||||
raw = bid["estimated_tokens"] / conf
|
||||
return raw * (1.15 - 0.3 * rep_score)
|
||||
|
||||
def award(self, auction_id: str) -> dict[str, Any]:
|
||||
auction = self._auctions[auction_id]
|
||||
if not auction.bids:
|
||||
raise RuntimeError(f"auction {auction_id} has no bids")
|
||||
if auction.awarded_to is not None:
|
||||
raise RuntimeError(f"auction {auction_id} already awarded")
|
||||
best = min(
|
||||
auction.bids,
|
||||
key=lambda b: self.score_bid(auction_id, b),
|
||||
)
|
||||
auction.awarded_to = best["agent"]
|
||||
return best
|
||||
|
||||
def mark_done(
|
||||
self,
|
||||
auction_id: str,
|
||||
answer: str,
|
||||
actual_tokens: int,
|
||||
correct: bool,
|
||||
) -> None:
|
||||
auction = self._auctions[auction_id]
|
||||
if auction.awarded_to is None:
|
||||
raise RuntimeError(f"auction {auction_id} not awarded yet")
|
||||
auction.result = {
|
||||
"answer": answer,
|
||||
"actual_tokens": int(actual_tokens),
|
||||
"correct": bool(correct),
|
||||
}
|
||||
key = (auction.awarded_to, auction.domain)
|
||||
entry = self._reputation.get(key) or ReputationEntry(
|
||||
agent=auction.awarded_to, domain=auction.domain
|
||||
)
|
||||
entry.runs += 1
|
||||
entry.tokens_spent += int(actual_tokens)
|
||||
if correct:
|
||||
entry.correct += 1
|
||||
self._reputation[key] = entry
|
||||
|
||||
# ---- reputation --------------------------------------------------------
|
||||
|
||||
def query_reputation(
|
||||
self, agent: str, domain: str
|
||||
) -> dict[str, Any]:
|
||||
rep = self._reputation.get((agent, domain))
|
||||
if rep is None:
|
||||
return {
|
||||
"agent": agent,
|
||||
"domain": domain,
|
||||
"runs": 0,
|
||||
"correct": 0,
|
||||
"tokens_spent": 0,
|
||||
"score": 0.5,
|
||||
}
|
||||
return {
|
||||
"agent": rep.agent,
|
||||
"domain": rep.domain,
|
||||
"runs": rep.runs,
|
||||
"correct": rep.correct,
|
||||
"tokens_spent": rep.tokens_spent,
|
||||
"score": rep.score(),
|
||||
}
|
||||
|
||||
def all_reputation(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
self.query_reputation(rep.agent, rep.domain)
|
||||
for rep in self._reputation.values()
|
||||
]
|
||||
|
||||
def auction(self, auction_id: str) -> Auction:
|
||||
return self._auctions[auction_id]
|
||||
@@ -0,0 +1,324 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Self-contained HTML report generator.
|
||||
|
||||
Renders a single HTML file with inline styles and an inline SVG scatter
|
||||
plot. No external assets, no CDN calls, nothing to fetch. Safe to open
|
||||
directly in a browser.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _esc(s: Any) -> str:
|
||||
return html.escape(str(s if s is not None else ""))
|
||||
|
||||
|
||||
def _scatter_svg(
|
||||
market_tokens: int,
|
||||
market_f1: float,
|
||||
baseline_tokens: int,
|
||||
baseline_f1: float,
|
||||
*,
|
||||
width: int = 520,
|
||||
height: int = 320,
|
||||
) -> str:
|
||||
pad_l, pad_r, pad_t, pad_b = 70, 30, 30, 50
|
||||
plot_w = width - pad_l - pad_r
|
||||
plot_h = height - pad_t - pad_b
|
||||
|
||||
max_tokens = max(market_tokens, baseline_tokens, 1)
|
||||
# Give a little headroom so points aren't on the axis.
|
||||
max_tokens_axis = max_tokens * 1.15
|
||||
min_tokens_axis = 0
|
||||
|
||||
def sx(tokens: float) -> float:
|
||||
frac = (tokens - min_tokens_axis) / max(
|
||||
max_tokens_axis - min_tokens_axis, 1
|
||||
)
|
||||
return pad_l + frac * plot_w
|
||||
|
||||
def sy(f1: float) -> float:
|
||||
# y=0 at top of plot, y=1 at bottom -> invert
|
||||
return pad_t + (1.0 - max(0.0, min(1.0, f1))) * plot_h
|
||||
|
||||
axis_color = "#555"
|
||||
grid_color = "#eee"
|
||||
market_color = "#2563eb"
|
||||
baseline_color = "#dc2626"
|
||||
|
||||
parts: list[str] = []
|
||||
parts.append(
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" '
|
||||
f'height="{height}" viewBox="0 0 {width} {height}" '
|
||||
f'role="img" aria-label="Pareto scatter: tokens vs F1">'
|
||||
)
|
||||
parts.append(
|
||||
f'<rect x="0" y="0" width="{width}" height="{height}" '
|
||||
f'fill="white"/>'
|
||||
)
|
||||
# Gridlines at F1 = 0, 0.25, 0.5, 0.75, 1.0
|
||||
for f in (0.0, 0.25, 0.5, 0.75, 1.0):
|
||||
y = sy(f)
|
||||
parts.append(
|
||||
f'<line x1="{pad_l}" y1="{y:.1f}" x2="{width-pad_r}" '
|
||||
f'y2="{y:.1f}" stroke="{grid_color}" stroke-width="1"/>'
|
||||
)
|
||||
parts.append(
|
||||
f'<text x="{pad_l-8}" y="{y+4:.1f}" font-family="sans-serif" '
|
||||
f'font-size="11" fill="{axis_color}" text-anchor="end">'
|
||||
f'{f:.2f}</text>'
|
||||
)
|
||||
# X-axis ticks
|
||||
for frac in (0.0, 0.25, 0.5, 0.75, 1.0):
|
||||
t_val = frac * max_tokens_axis
|
||||
x = sx(t_val)
|
||||
parts.append(
|
||||
f'<line x1="{x:.1f}" y1="{height-pad_b}" x2="{x:.1f}" '
|
||||
f'y2="{height-pad_b+4}" stroke="{axis_color}"/>'
|
||||
)
|
||||
parts.append(
|
||||
f'<text x="{x:.1f}" y="{height-pad_b+18}" '
|
||||
f'font-family="sans-serif" font-size="11" fill="{axis_color}" '
|
||||
f'text-anchor="middle">{int(t_val)}</text>'
|
||||
)
|
||||
# Axis lines
|
||||
parts.append(
|
||||
f'<line x1="{pad_l}" y1="{pad_t}" x2="{pad_l}" '
|
||||
f'y2="{height-pad_b}" stroke="{axis_color}"/>'
|
||||
)
|
||||
parts.append(
|
||||
f'<line x1="{pad_l}" y1="{height-pad_b}" x2="{width-pad_r}" '
|
||||
f'y2="{height-pad_b}" stroke="{axis_color}"/>'
|
||||
)
|
||||
# Axis labels
|
||||
parts.append(
|
||||
f'<text x="{width/2:.1f}" y="{height-10}" '
|
||||
f'font-family="sans-serif" font-size="12" fill="{axis_color}" '
|
||||
f'text-anchor="middle">tokens</text>'
|
||||
)
|
||||
parts.append(
|
||||
f'<text x="15" y="{height/2:.1f}" font-family="sans-serif" '
|
||||
f'font-size="12" fill="{axis_color}" text-anchor="middle" '
|
||||
f'transform="rotate(-90 15 {height/2:.1f})">F1</text>'
|
||||
)
|
||||
# Baseline point
|
||||
bx, by = sx(baseline_tokens), sy(baseline_f1)
|
||||
parts.append(
|
||||
f'<circle cx="{bx:.1f}" cy="{by:.1f}" r="7" '
|
||||
f'fill="{baseline_color}"/>'
|
||||
)
|
||||
parts.append(
|
||||
f'<text x="{bx+10:.1f}" y="{by+4:.1f}" font-family="sans-serif" '
|
||||
f'font-size="11" fill="{baseline_color}">baseline</text>'
|
||||
)
|
||||
# Market point
|
||||
mx, my = sx(market_tokens), sy(market_f1)
|
||||
parts.append(
|
||||
f'<circle cx="{mx:.1f}" cy="{my:.1f}" r="7" '
|
||||
f'fill="{market_color}"/>'
|
||||
)
|
||||
parts.append(
|
||||
f'<text x="{mx+10:.1f}" y="{my+4:.1f}" font-family="sans-serif" '
|
||||
f'font-size="11" fill="{market_color}">marketplace</text>'
|
||||
)
|
||||
|
||||
parts.append("</svg>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
CSS = """\
|
||||
body { font-family: -apple-system, system-ui, sans-serif;
|
||||
max-width: 960px; margin: 2rem auto; padding: 0 1rem;
|
||||
color: #1f2937; line-height: 1.55; }
|
||||
h1, h2, h3 { color: #111827; }
|
||||
h1 { border-bottom: 2px solid #2563eb; padding-bottom: .4rem; }
|
||||
.verdict-pass { display: inline-block; background: #dcfce7;
|
||||
color: #166534; padding: .3rem .8rem; border-radius: 6px;
|
||||
font-weight: 600; }
|
||||
.verdict-fail { display: inline-block; background: #fee2e2;
|
||||
color: #991b1b; padding: .3rem .8rem; border-radius: 6px;
|
||||
font-weight: 600; }
|
||||
table { border-collapse: collapse; margin: .8rem 0; width: 100%; }
|
||||
th, td { border: 1px solid #e5e7eb; padding: .4rem .6rem;
|
||||
text-align: left; vertical-align: top; }
|
||||
th { background: #f9fafb; }
|
||||
pre, code { background: #f3f4f6; border-radius: 4px;
|
||||
padding: .1rem .4rem; font-size: .9rem; }
|
||||
pre { padding: .8rem; white-space: pre-wrap; word-break: break-word; }
|
||||
.card { border: 1px solid #e5e7eb; border-radius: 8px;
|
||||
padding: 1rem 1.2rem; margin: 1rem 0; background: #fff; }
|
||||
.kv { display: grid; grid-template-columns: 180px 1fr; gap: .3rem .8rem; }
|
||||
.small { color: #6b7280; font-size: .88rem; }
|
||||
"""
|
||||
|
||||
|
||||
def render_report(data: dict[str, Any], out_path: Path) -> None:
|
||||
verdict = data.get("pareto", {})
|
||||
is_pass = verdict.get("verdict") == "PASS"
|
||||
verdict_html = (
|
||||
'<span class="verdict-pass">PASS — strictly northwest</span>'
|
||||
if is_pass
|
||||
else '<span class="verdict-fail">FAIL — not dominating baseline</span>'
|
||||
)
|
||||
|
||||
bids_rows: list[str] = []
|
||||
for b in data.get("bids", []):
|
||||
conf_str = "{:.2f}".format(b.get("confidence", 0) or 0)
|
||||
bids_rows.append(
|
||||
f"<tr><td>{_esc(b.get('agent'))}</td>"
|
||||
f"<td>{_esc(b.get('estimated_tokens'))}</td>"
|
||||
f"<td>{_esc(conf_str)}</td>"
|
||||
f"<td>{_esc(b.get('approach'))}</td></tr>"
|
||||
)
|
||||
bids_table = "\n".join(bids_rows) or (
|
||||
"<tr><td colspan=4>no bids</td></tr>"
|
||||
)
|
||||
|
||||
decomp_rows: list[str] = []
|
||||
for i, sub in enumerate(data.get("decomposition", []) or [], start=1):
|
||||
decomp_rows.append(
|
||||
f"<tr><td>{i}</td><td>{_esc(sub.get('question'))}</td>"
|
||||
f"<td>{_esc(sub.get('answer'))}</td></tr>"
|
||||
)
|
||||
decomp_table = "\n".join(decomp_rows) or (
|
||||
"<tr><td colspan=3>(none)</td></tr>"
|
||||
)
|
||||
|
||||
rep_rows: list[str] = []
|
||||
for rep in data.get("reputation", []) or []:
|
||||
score_str = "{:.3f}".format(rep.get("score", 0) or 0)
|
||||
rep_rows.append(
|
||||
f"<tr><td>{_esc(rep.get('agent'))}</td>"
|
||||
f"<td>{_esc(rep.get('domain'))}</td>"
|
||||
f"<td>{_esc(rep.get('runs'))}</td>"
|
||||
f"<td>{_esc(rep.get('correct'))}</td>"
|
||||
f"<td>{_esc(rep.get('tokens_spent'))}</td>"
|
||||
f"<td>{_esc(score_str)}</td></tr>"
|
||||
)
|
||||
rep_table = "\n".join(rep_rows) or (
|
||||
"<tr><td colspan=6>(empty)</td></tr>"
|
||||
)
|
||||
|
||||
svg = _scatter_svg(
|
||||
market_tokens=int(data.get("market", {}).get("tokens", 0)),
|
||||
market_f1=float(data.get("market", {}).get("f1", 0.0)),
|
||||
baseline_tokens=int(data.get("baseline", {}).get("tokens", 0)),
|
||||
baseline_f1=float(data.get("baseline", {}).get("f1", 0.0)),
|
||||
)
|
||||
|
||||
market = data.get("market", {})
|
||||
baseline = data.get("baseline", {})
|
||||
|
||||
market_f1_str = "{:.3f}".format(verdict.get("market_f1", 0) or 0)
|
||||
baseline_f1_str = "{:.3f}".format(verdict.get("baseline_f1", 0) or 0)
|
||||
f1_delta_str = "{:.3f}".format(verdict.get("f1_delta", 0) or 0)
|
||||
market_run_f1_str = "{:.3f}".format(market.get("f1", 0) or 0)
|
||||
baseline_run_f1_str = "{:.3f}".format(baseline.get("f1", 0) or 0)
|
||||
|
||||
html_doc = f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>MuSiQue MAS Benchmark — {_esc(data.get('question_id', ''))}</title>
|
||||
<style>{CSS}</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>MuSiQue MAS Benchmark Report</h1>
|
||||
<p class="small">
|
||||
Mode: <code>{_esc(data.get('mode', ''))}</code>
|
||||
· Question: <code>{_esc(data.get('question_id', ''))}</code>
|
||||
· Dry-run: <code>{_esc(data.get('dry_run', False))}</code>
|
||||
</p>
|
||||
|
||||
<div class="card">
|
||||
<h2>Verdict</h2>
|
||||
<p>{verdict_html}</p>
|
||||
<div class="kv">
|
||||
<div>Market tokens</div><div>{_esc(verdict.get('market_tokens'))}</div>
|
||||
<div>Market F1</div><div>{_esc(market_f1_str)}</div>
|
||||
<div>Baseline tokens</div><div>{_esc(verdict.get('baseline_tokens'))}</div>
|
||||
<div>Baseline F1</div><div>{_esc(baseline_f1_str)}</div>
|
||||
<div>Tokens delta</div><div>{_esc(verdict.get('tokens_delta'))}</div>
|
||||
<div>F1 delta</div><div>{_esc(f1_delta_str)}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Pareto plot</h2>
|
||||
{svg}
|
||||
<p class="small">
|
||||
Lower-right = expensive and wrong. Upper-left = cheap and correct.
|
||||
Marketplace must sit strictly northwest of baseline to pass.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Question</h2>
|
||||
<p><strong>{_esc(data.get('question', ''))}</strong></p>
|
||||
<p>Gold answer: <code>{_esc(data.get('gold_answer', ''))}</code></p>
|
||||
|
||||
<h3>Gold decomposition</h3>
|
||||
<table>
|
||||
<tr><th>#</th><th>Sub-question</th><th>Sub-answer</th></tr>
|
||||
{decomp_table}
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Auction</h2>
|
||||
<p>Domain: <code>{_esc(data.get('domain', ''))}</code>
|
||||
· Budget: <code>{_esc(data.get('max_budget_tokens', ''))}</code>
|
||||
· Awarded to: <code>{_esc(data.get('awarded_to', ''))}</code></p>
|
||||
<h3>Bids received</h3>
|
||||
<table>
|
||||
<tr><th>Agent</th><th>Est. tokens</th><th>Confidence</th><th>Approach</th></tr>
|
||||
{bids_table}
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Marketplace run</h2>
|
||||
<div class="kv">
|
||||
<div>Winning agent</div><div>{_esc(market.get('agent'))}</div>
|
||||
<div>Model</div><div>{_esc(market.get('model'))}</div>
|
||||
<div>Tokens</div><div>{_esc(market.get('tokens'))}</div>
|
||||
<div>F1</div><div>{_esc(market_run_f1_str)}</div>
|
||||
<div>Answer</div><div><code>{_esc(market.get('answer'))}</code></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Single-agent baseline</h2>
|
||||
<div class="kv">
|
||||
<div>Model</div><div>{_esc(baseline.get('model'))}</div>
|
||||
<div>Tokens</div><div>{_esc(baseline.get('tokens'))}</div>
|
||||
<div>F1</div><div>{_esc(baseline_run_f1_str)}</div>
|
||||
<div>Answer</div><div><code>{_esc(baseline.get('answer'))}</code></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Reputation ledger (post-run)</h2>
|
||||
<table>
|
||||
<tr><th>Agent</th><th>Domain</th><th>Runs</th><th>Correct</th>
|
||||
<th>Tokens</th><th>Score</th></tr>
|
||||
{rep_table}
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<p class="small">
|
||||
Generated by <code>benchmark/report.py</code>.
|
||||
Marketplace primitives are currently stubbed in-process — see
|
||||
<code>benchmark/marketplace.py</code> for the migration plan to the
|
||||
real 016 SynapBus MCP tools.
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(html_doc, encoding="utf-8")
|
||||
@@ -0,0 +1,2 @@
|
||||
anthropic>=0.40.0
|
||||
requests>=2.31.0
|
||||
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Main entry point for the MuSiQue MAS benchmark.
|
||||
|
||||
Usage::
|
||||
|
||||
python benchmark/run.py --mode single-shot --question q1
|
||||
python benchmark/run.py --mode single-shot --question q1 --dry-run
|
||||
|
||||
Flow (single-shot):
|
||||
1. Load trio.jsonl, find the requested question (by short_id).
|
||||
2. Marketplace run:
|
||||
a. post_auction(task, domain, max_budget)
|
||||
b. each agent in the pool submits a bid
|
||||
c. marketplace awards best bid
|
||||
d. winner executes (Anthropic call or dry-run stub)
|
||||
e. marketplace.mark_done records reputation
|
||||
3. Baseline run: one Sonnet call with all distractors.
|
||||
4. Score both, compute Pareto verdict.
|
||||
5. Write results/latest.json and results/latest.html.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Allow running as ``python benchmark/run.py`` from the repo root.
|
||||
_HERE = Path(__file__).resolve().parent
|
||||
if str(_HERE) not in sys.path:
|
||||
sys.path.insert(0, str(_HERE))
|
||||
|
||||
from agents import default_pool # noqa: E402
|
||||
from baseline import run_baseline, BASELINE_MODEL # noqa: E402
|
||||
from marketplace import Marketplace # noqa: E402
|
||||
from report import render_report # noqa: E402
|
||||
from score import best_f1_against_aliases, pareto_verdict # noqa: E402
|
||||
|
||||
|
||||
TRIO_FILE = _HERE / "trio.jsonl"
|
||||
RESULTS_DIR = _HERE / "results"
|
||||
DEFAULT_DOMAIN = "multi-hop-qa"
|
||||
DEFAULT_BUDGET = 50_000
|
||||
|
||||
|
||||
def _load_trio() -> list[dict[str, Any]]:
|
||||
if not TRIO_FILE.exists():
|
||||
raise SystemExit(
|
||||
f"[run] trio.jsonl not found at {TRIO_FILE}. "
|
||||
"Run curate.py first."
|
||||
)
|
||||
out: list[dict[str, Any]] = []
|
||||
with open(TRIO_FILE, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
out.append(json.loads(line))
|
||||
return out
|
||||
|
||||
|
||||
def _pick_question(
|
||||
trio: list[dict[str, Any]], want: str
|
||||
) -> dict[str, Any]:
|
||||
for rec in trio:
|
||||
if rec.get("short_id") == want or rec.get("id") == want:
|
||||
return rec
|
||||
raise SystemExit(
|
||||
f"[run] question {want!r} not found. Available: "
|
||||
+ ", ".join(r.get("short_id", r.get("id", "?")) for r in trio)
|
||||
)
|
||||
|
||||
|
||||
def single_shot(
|
||||
question: str,
|
||||
*,
|
||||
dry_run: bool,
|
||||
verbose: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
trio = _load_trio()
|
||||
rec = _pick_question(trio, question)
|
||||
|
||||
task = {
|
||||
"question": rec["question"],
|
||||
"short_id": rec.get("short_id"),
|
||||
}
|
||||
paragraphs = rec.get("paragraphs", []) or []
|
||||
gold_answer = rec.get("answer", "")
|
||||
aliases = rec.get("answer_aliases", []) or []
|
||||
|
||||
market = Marketplace()
|
||||
pool = default_pool()
|
||||
|
||||
if verbose:
|
||||
print(f"[run] question {rec.get('short_id')}: {rec['question']!r}")
|
||||
print(
|
||||
f"[run] agents: "
|
||||
+ ", ".join(f"{a.name}({a.model})" for a in pool)
|
||||
)
|
||||
print(f"[run] paragraphs: {len(paragraphs)}")
|
||||
|
||||
# --- Marketplace path ------------------------------------------------
|
||||
auction_id = market.post_auction(
|
||||
task=task,
|
||||
domain=DEFAULT_DOMAIN,
|
||||
max_budget_tokens=DEFAULT_BUDGET,
|
||||
)
|
||||
if verbose:
|
||||
print(f"[run] posted auction {auction_id}")
|
||||
|
||||
for agent in pool:
|
||||
bid = agent.bid(task)
|
||||
market.bid(
|
||||
auction_id=auction_id,
|
||||
agent=agent.name,
|
||||
estimated_tokens=bid.estimated_tokens,
|
||||
confidence=bid.confidence,
|
||||
approach=bid.approach,
|
||||
)
|
||||
if verbose:
|
||||
print(
|
||||
f"[run] bid {agent.name}: "
|
||||
f"est={bid.estimated_tokens} conf={bid.confidence:.2f}"
|
||||
)
|
||||
|
||||
winning_bid = market.award(auction_id)
|
||||
winner_name = winning_bid["agent"]
|
||||
winner = next(a for a in pool if a.name == winner_name)
|
||||
if verbose:
|
||||
print(f"[run] awarded to {winner_name}")
|
||||
|
||||
start = time.time()
|
||||
result = winner.execute(
|
||||
task=task,
|
||||
paragraphs=paragraphs,
|
||||
dry_run=dry_run,
|
||||
max_budget_tokens=DEFAULT_BUDGET,
|
||||
)
|
||||
market_wall = time.time() - start
|
||||
|
||||
market_f1 = best_f1_against_aliases(
|
||||
result.answer, gold_answer, aliases
|
||||
)
|
||||
market.mark_done(
|
||||
auction_id=auction_id,
|
||||
answer=result.answer,
|
||||
actual_tokens=result.actual_tokens,
|
||||
correct=market_f1 >= 0.5,
|
||||
)
|
||||
if verbose:
|
||||
print(
|
||||
f"[run] market answer: {result.answer!r} "
|
||||
f"(tokens={result.actual_tokens}, f1={market_f1:.3f})"
|
||||
)
|
||||
|
||||
# --- Baseline path ---------------------------------------------------
|
||||
start = time.time()
|
||||
baseline = run_baseline(
|
||||
question=rec["question"],
|
||||
paragraphs=paragraphs,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
baseline_wall = time.time() - start
|
||||
baseline_f1 = best_f1_against_aliases(
|
||||
baseline["answer"], gold_answer, aliases
|
||||
)
|
||||
if verbose:
|
||||
print(
|
||||
f"[run] baseline answer: {baseline['answer']!r} "
|
||||
f"(tokens={baseline['tokens']}, f1={baseline_f1:.3f})"
|
||||
)
|
||||
|
||||
verdict = pareto_verdict(
|
||||
market_tokens=result.actual_tokens,
|
||||
market_f1=market_f1,
|
||||
baseline_tokens=baseline["tokens"],
|
||||
baseline_f1=baseline_f1,
|
||||
)
|
||||
if verbose:
|
||||
print(f"[run] PARETO VERDICT: {verdict['verdict']}")
|
||||
|
||||
return {
|
||||
"mode": "single-shot",
|
||||
"dry_run": dry_run,
|
||||
"question_id": rec.get("short_id"),
|
||||
"musique_id": rec.get("id"),
|
||||
"question": rec["question"],
|
||||
"gold_answer": gold_answer,
|
||||
"decomposition": rec.get("decomposition", []),
|
||||
"domain": DEFAULT_DOMAIN,
|
||||
"max_budget_tokens": DEFAULT_BUDGET,
|
||||
"awarded_to": winner_name,
|
||||
"bids": market.list_bids(auction_id),
|
||||
"market": {
|
||||
"agent": winner_name,
|
||||
"model": winner.model,
|
||||
"tokens": result.actual_tokens,
|
||||
"answer": result.answer,
|
||||
"f1": market_f1,
|
||||
"wall_seconds": market_wall,
|
||||
"raw_text": result.raw_text,
|
||||
},
|
||||
"baseline": {
|
||||
"model": baseline.get("model", BASELINE_MODEL),
|
||||
"tokens": baseline["tokens"],
|
||||
"answer": baseline["answer"],
|
||||
"f1": baseline_f1,
|
||||
"wall_seconds": baseline_wall,
|
||||
"raw_text": baseline.get("raw_text", ""),
|
||||
},
|
||||
"pareto": verdict,
|
||||
"reputation": market.all_reputation(),
|
||||
}
|
||||
|
||||
|
||||
def _write_outputs(result: dict[str, Any]) -> tuple[Path, Path]:
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
json_path = RESULTS_DIR / "latest.json"
|
||||
html_path = RESULTS_DIR / "latest.html"
|
||||
# Trim raw_text from json to keep it small and readable.
|
||||
trimmed = dict(result)
|
||||
for key in ("market", "baseline"):
|
||||
section = dict(trimmed.get(key, {}))
|
||||
if "raw_text" in section:
|
||||
section["raw_text"] = (section["raw_text"] or "")[:2000]
|
||||
trimmed[key] = section
|
||||
json_path.write_text(
|
||||
json.dumps(trimmed, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
render_report(result, html_path)
|
||||
return json_path, html_path
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="MuSiQue multi-agent benchmark harness"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=["single-shot"],
|
||||
default="single-shot",
|
||||
help="Run mode (only single-shot is implemented in MVP)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--question",
|
||||
default="q1",
|
||||
help="Question short_id from trio.jsonl (q1/q2/q3)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Skip real Anthropic API calls; use stub responses",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.mode != "single-shot":
|
||||
print(f"[run] mode {args.mode} not implemented in MVP", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
result = single_shot(
|
||||
question=args.question,
|
||||
dry_run=args.dry_run,
|
||||
verbose=True,
|
||||
)
|
||||
json_path, html_path = _write_outputs(result)
|
||||
print(f"[run] wrote {json_path}")
|
||||
print(f"[run] wrote {html_path}")
|
||||
print(f"[run] verdict: {result['pareto']['verdict']}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Scoring utilities for the MuSiQue benchmark.
|
||||
|
||||
- Normalized exact-match F1 (SQuAD-style): lowercase, strip articles,
|
||||
strip punctuation, collapse whitespace.
|
||||
- Pareto verdict: the marketplace point is strictly northwest of the
|
||||
baseline iff it uses fewer tokens AND has F1 >= baseline, with at
|
||||
least one of those strict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import string
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
_ARTICLE_RE = re.compile(r"\b(a|an|the)\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
if text is None:
|
||||
return ""
|
||||
text = text.lower()
|
||||
text = _ARTICLE_RE.sub(" ", text)
|
||||
text = "".join(ch for ch in text if ch not in string.punctuation)
|
||||
text = " ".join(text.split())
|
||||
return text
|
||||
|
||||
|
||||
def f1(prediction: str, gold: str) -> float:
|
||||
pred_tokens = normalize(prediction).split()
|
||||
gold_tokens = normalize(gold).split()
|
||||
if not pred_tokens and not gold_tokens:
|
||||
return 1.0
|
||||
if not pred_tokens or not gold_tokens:
|
||||
return 0.0
|
||||
common = Counter(pred_tokens) & Counter(gold_tokens)
|
||||
overlap = sum(common.values())
|
||||
if overlap == 0:
|
||||
return 0.0
|
||||
precision = overlap / len(pred_tokens)
|
||||
recall = overlap / len(gold_tokens)
|
||||
return 2 * precision * recall / (precision + recall)
|
||||
|
||||
|
||||
def exact_match(prediction: str, gold: str) -> bool:
|
||||
return normalize(prediction) == normalize(gold)
|
||||
|
||||
|
||||
def best_f1_against_aliases(
|
||||
prediction: str, gold: str, aliases: list[str] | None = None
|
||||
) -> float:
|
||||
candidates = [gold] + list(aliases or [])
|
||||
return max(f1(prediction, c) for c in candidates if c is not None)
|
||||
|
||||
|
||||
def pareto_verdict(
|
||||
market_tokens: int,
|
||||
market_f1: float,
|
||||
baseline_tokens: int,
|
||||
baseline_f1: float,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Strictly northwest of baseline: fewer tokens AND higher-or-equal F1,
|
||||
with at least one strict inequality.
|
||||
"""
|
||||
tokens_better = market_tokens < baseline_tokens
|
||||
quality_atleast = market_f1 >= baseline_f1
|
||||
quality_better = market_f1 > baseline_f1
|
||||
|
||||
strictly_nw = (
|
||||
(tokens_better and quality_atleast)
|
||||
or (quality_better and market_tokens <= baseline_tokens)
|
||||
)
|
||||
return {
|
||||
"verdict": "PASS" if strictly_nw else "FAIL",
|
||||
"strictly_northwest": strictly_nw,
|
||||
"market_tokens": int(market_tokens),
|
||||
"market_f1": float(market_f1),
|
||||
"baseline_tokens": int(baseline_tokens),
|
||||
"baseline_f1": float(baseline_f1),
|
||||
"tokens_delta": int(market_tokens - baseline_tokens),
|
||||
"f1_delta": float(market_f1 - baseline_f1),
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MuSiQue dataset downloader.
|
||||
|
||||
Downloads ``musique_v1.0.zip`` from the canonical source used by the
|
||||
upstream project (https://github.com/StonyBrookNLP/musique). The zip is
|
||||
hosted on Google Drive (file id ``1tGdADlNjWFaHLeZZGShh2IRcpO6Lv24h``);
|
||||
this mirrors the behavior of the project's ``download_data.sh`` which
|
||||
uses ``gdown`` under the hood.
|
||||
|
||||
Idempotent — skips download if the target dev-set jsonl already exists.
|
||||
Run: ``python benchmark/setup.py``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
GDRIVE_FILE_ID = "1tGdADlNjWFaHLeZZGShh2IRcpO6Lv24h"
|
||||
GDRIVE_URL = "https://docs.google.com/uc?export=download"
|
||||
|
||||
DATA_DIR = Path(__file__).resolve().parent / "data"
|
||||
ZIP_PATH = DATA_DIR / "musique_v1.0.zip"
|
||||
TARGET_FILE = DATA_DIR / "musique_ans_v1.0_dev.jsonl"
|
||||
|
||||
|
||||
def _write_stream(resp: requests.Response, dest: Path) -> int:
|
||||
total = int(resp.headers.get("Content-Length", 0))
|
||||
downloaded = 0
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(dest, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=1024 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
if total:
|
||||
pct = 100.0 * downloaded / total
|
||||
print(
|
||||
f"\r downloading: {downloaded/1e6:6.1f} MB "
|
||||
f"/ {total/1e6:6.1f} MB ({pct:5.1f}%)",
|
||||
end="",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print("", file=sys.stderr)
|
||||
return downloaded
|
||||
|
||||
|
||||
def _download_gdrive(file_id: str, dest: Path) -> bool:
|
||||
"""
|
||||
Download a large file from Google Drive, handling the virus-scan
|
||||
confirmation page that Drive injects for anything over ~100 MB.
|
||||
"""
|
||||
session = requests.Session()
|
||||
try:
|
||||
resp = session.get(
|
||||
GDRIVE_URL,
|
||||
params={"id": file_id, "export": "download"},
|
||||
stream=True,
|
||||
timeout=60,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
print(f" -> request failed: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
# Case 1: Drive returns the file directly (small file or cached).
|
||||
ctype = resp.headers.get("Content-Type", "")
|
||||
if "text/html" not in ctype.lower():
|
||||
_write_stream(resp, dest)
|
||||
return dest.exists() and dest.stat().st_size > 0
|
||||
|
||||
# Case 2: HTML confirmation page. Extract the confirm token and/or
|
||||
# the form action URL.
|
||||
html = resp.text
|
||||
# Newer Drive flow: a <form ...> with all the params we need.
|
||||
form_match = re.search(
|
||||
r'<form[^>]*id="download-form"[^>]*action="([^"]+)"', html
|
||||
)
|
||||
if form_match:
|
||||
action = form_match.group(1).replace("&", "&")
|
||||
params = dict(
|
||||
re.findall(
|
||||
r'name="([^"]+)"[^>]*value="([^"]+)"', html
|
||||
)
|
||||
)
|
||||
try:
|
||||
resp2 = session.get(action, params=params, stream=True, timeout=120)
|
||||
if resp2.status_code == 200:
|
||||
_write_stream(resp2, dest)
|
||||
return dest.exists() and dest.stat().st_size > 0
|
||||
except requests.RequestException as exc:
|
||||
print(f" -> form post failed: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
# Older flow: confirm cookie token.
|
||||
token = None
|
||||
for k, v in session.cookies.items():
|
||||
if k.startswith("download_warning"):
|
||||
token = v
|
||||
break
|
||||
if token is None:
|
||||
m = re.search(r'confirm=([0-9A-Za-z_-]+)', html)
|
||||
if m:
|
||||
token = m.group(1)
|
||||
if token:
|
||||
try:
|
||||
resp3 = session.get(
|
||||
GDRIVE_URL,
|
||||
params={
|
||||
"id": file_id,
|
||||
"export": "download",
|
||||
"confirm": token,
|
||||
},
|
||||
stream=True,
|
||||
timeout=120,
|
||||
)
|
||||
if resp3.status_code == 200:
|
||||
_write_stream(resp3, dest)
|
||||
return dest.exists() and dest.stat().st_size > 0
|
||||
except requests.RequestException as exc:
|
||||
print(f" -> confirm fetch failed: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
print(" -> could not navigate Google Drive download flow", file=sys.stderr)
|
||||
return False
|
||||
|
||||
|
||||
def _extract(zip_path: Path, out_dir: Path) -> None:
|
||||
"""Extract the dev set jsonl from the zip."""
|
||||
wanted_suffixes = (
|
||||
"musique_ans_v1.0_dev.jsonl",
|
||||
"musique_ans_v1.0_train.jsonl",
|
||||
)
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
members = zf.namelist()
|
||||
extracted_any = False
|
||||
for m in members:
|
||||
base = os.path.basename(m)
|
||||
if base in wanted_suffixes:
|
||||
with zf.open(m) as src, open(out_dir / base, "wb") as dst:
|
||||
dst.write(src.read())
|
||||
print(f" extracted: {base}")
|
||||
extracted_any = True
|
||||
if not extracted_any:
|
||||
# Fall back: extract everything so a human can inspect.
|
||||
zf.extractall(out_dir)
|
||||
print(
|
||||
" could not find canonical filenames; extracted all",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if TARGET_FILE.exists():
|
||||
size = TARGET_FILE.stat().st_size
|
||||
print(f"[setup] already present: {TARGET_FILE} ({size/1e6:.1f} MB)")
|
||||
return 0
|
||||
|
||||
print(f"[setup] downloading Google Drive file id {GDRIVE_FILE_ID}")
|
||||
ok = _download_gdrive(GDRIVE_FILE_ID, ZIP_PATH)
|
||||
|
||||
if not ok:
|
||||
print(
|
||||
"[setup] ERROR: failed to download MuSiQue. Please download "
|
||||
"manually from "
|
||||
f"https://drive.google.com/file/d/{GDRIVE_FILE_ID}/view "
|
||||
f"and place the zip at {ZIP_PATH}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
print(f"[setup] extracting {ZIP_PATH}")
|
||||
_extract(ZIP_PATH, DATA_DIR)
|
||||
|
||||
if not TARGET_FILE.exists():
|
||||
print(
|
||||
f"[setup] WARNING: {TARGET_FILE.name} not found after extract. "
|
||||
f"Listing {DATA_DIR}:",
|
||||
file=sys.stderr,
|
||||
)
|
||||
for p in sorted(DATA_DIR.iterdir()):
|
||||
print(f" - {p.name}", file=sys.stderr)
|
||||
return 3
|
||||
|
||||
print(f"[setup] ready: {TARGET_FILE}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user