#!/usr/bin/env python3 """ Self-contained HTML report generator. Renders a single HTML file with inline styles and an inline SVG scatter plot. No external assets, no CDN calls, nothing to fetch. Safe to open directly in a browser. """ from __future__ import annotations import html from pathlib import Path from typing import Any def _esc(s: Any) -> str: return html.escape(str(s if s is not None else "")) def _scatter_svg( market_tokens: int, market_f1: float, baseline_tokens: int, baseline_f1: float, *, width: int = 520, height: int = 320, ) -> str: pad_l, pad_r, pad_t, pad_b = 70, 30, 30, 50 plot_w = width - pad_l - pad_r plot_h = height - pad_t - pad_b max_tokens = max(market_tokens, baseline_tokens, 1) # Give a little headroom so points aren't on the axis. max_tokens_axis = max_tokens * 1.15 min_tokens_axis = 0 def sx(tokens: float) -> float: frac = (tokens - min_tokens_axis) / max( max_tokens_axis - min_tokens_axis, 1 ) return pad_l + frac * plot_w def sy(f1: float) -> float: # y=0 at top of plot, y=1 at bottom -> invert return pad_t + (1.0 - max(0.0, min(1.0, f1))) * plot_h axis_color = "#555" grid_color = "#eee" market_color = "#2563eb" baseline_color = "#dc2626" parts: list[str] = [] parts.append( f'' ) parts.append( f'' ) # Gridlines at F1 = 0, 0.25, 0.5, 0.75, 1.0 for f in (0.0, 0.25, 0.5, 0.75, 1.0): y = sy(f) parts.append( f'' ) parts.append( f'' f'{f:.2f}' ) # X-axis ticks for frac in (0.0, 0.25, 0.5, 0.75, 1.0): t_val = frac * max_tokens_axis x = sx(t_val) parts.append( f'' ) parts.append( f'{int(t_val)}' ) # Axis lines parts.append( f'' ) parts.append( f'' ) # Axis labels parts.append( f'tokens' ) parts.append( f'F1' ) # Baseline point bx, by = sx(baseline_tokens), sy(baseline_f1) parts.append( f'' ) parts.append( f'baseline' ) # Market point mx, my = sx(market_tokens), sy(market_f1) parts.append( f'' ) parts.append( f'marketplace' ) parts.append("") return "".join(parts) CSS = """\ body { font-family: -apple-system, system-ui, sans-serif; max-width: 960px; margin: 2rem auto; padding: 0 1rem; color: #1f2937; line-height: 1.55; } h1, h2, h3 { color: #111827; } h1 { border-bottom: 2px solid #2563eb; padding-bottom: .4rem; } .verdict-pass { display: inline-block; background: #dcfce7; color: #166534; padding: .3rem .8rem; border-radius: 6px; font-weight: 600; } .verdict-fail { display: inline-block; background: #fee2e2; color: #991b1b; padding: .3rem .8rem; border-radius: 6px; font-weight: 600; } table { border-collapse: collapse; margin: .8rem 0; width: 100%; } th, td { border: 1px solid #e5e7eb; padding: .4rem .6rem; text-align: left; vertical-align: top; } th { background: #f9fafb; } pre, code { background: #f3f4f6; border-radius: 4px; padding: .1rem .4rem; font-size: .9rem; } pre { padding: .8rem; white-space: pre-wrap; word-break: break-word; } .card { border: 1px solid #e5e7eb; border-radius: 8px; padding: 1rem 1.2rem; margin: 1rem 0; background: #fff; } .kv { display: grid; grid-template-columns: 180px 1fr; gap: .3rem .8rem; } .small { color: #6b7280; font-size: .88rem; } """ def render_report(data: dict[str, Any], out_path: Path) -> None: verdict = data.get("pareto", {}) is_pass = verdict.get("verdict") == "PASS" verdict_html = ( 'PASS — strictly northwest' if is_pass else 'FAIL — not dominating baseline' ) bids_rows: list[str] = [] for b in data.get("bids", []): conf_str = "{:.2f}".format(b.get("confidence", 0) or 0) bids_rows.append( f"{_esc(b.get('agent'))}" f"{_esc(b.get('estimated_tokens'))}" f"{_esc(conf_str)}" f"{_esc(b.get('approach'))}" ) bids_table = "\n".join(bids_rows) or ( "no bids" ) decomp_rows: list[str] = [] for i, sub in enumerate(data.get("decomposition", []) or [], start=1): decomp_rows.append( f"{i}{_esc(sub.get('question'))}" f"{_esc(sub.get('answer'))}" ) decomp_table = "\n".join(decomp_rows) or ( "(none)" ) rep_rows: list[str] = [] for rep in data.get("reputation", []) or []: score_str = "{:.3f}".format(rep.get("score", 0) or 0) rep_rows.append( f"{_esc(rep.get('agent'))}" f"{_esc(rep.get('domain'))}" f"{_esc(rep.get('runs'))}" f"{_esc(rep.get('correct'))}" f"{_esc(rep.get('tokens_spent'))}" f"{_esc(score_str)}" ) rep_table = "\n".join(rep_rows) or ( "(empty)" ) svg = _scatter_svg( market_tokens=int(data.get("market", {}).get("tokens", 0)), market_f1=float(data.get("market", {}).get("f1", 0.0)), baseline_tokens=int(data.get("baseline", {}).get("tokens", 0)), baseline_f1=float(data.get("baseline", {}).get("f1", 0.0)), ) market = data.get("market", {}) baseline = data.get("baseline", {}) market_f1_str = "{:.3f}".format(verdict.get("market_f1", 0) or 0) baseline_f1_str = "{:.3f}".format(verdict.get("baseline_f1", 0) or 0) f1_delta_str = "{:.3f}".format(verdict.get("f1_delta", 0) or 0) market_run_f1_str = "{:.3f}".format(market.get("f1", 0) or 0) baseline_run_f1_str = "{:.3f}".format(baseline.get("f1", 0) or 0) html_doc = f""" MuSiQue MAS Benchmark — {_esc(data.get('question_id', ''))}

MuSiQue MAS Benchmark Report

Mode: {_esc(data.get('mode', ''))} · Question: {_esc(data.get('question_id', ''))} · Dry-run: {_esc(data.get('dry_run', False))}

Verdict

{verdict_html}

Market tokens
{_esc(verdict.get('market_tokens'))}
Market F1
{_esc(market_f1_str)}
Baseline tokens
{_esc(verdict.get('baseline_tokens'))}
Baseline F1
{_esc(baseline_f1_str)}
Tokens delta
{_esc(verdict.get('tokens_delta'))}
F1 delta
{_esc(f1_delta_str)}

Pareto plot

{svg}

Lower-right = expensive and wrong. Upper-left = cheap and correct. Marketplace must sit strictly northwest of baseline to pass.

Question

{_esc(data.get('question', ''))}

Gold answer: {_esc(data.get('gold_answer', ''))}

Gold decomposition

{decomp_table}
#Sub-questionSub-answer

Auction

Domain: {_esc(data.get('domain', ''))} · Budget: {_esc(data.get('max_budget_tokens', ''))} · Awarded to: {_esc(data.get('awarded_to', ''))}

Bids received

{bids_table}
AgentEst. tokensConfidenceApproach

Marketplace run

Winning agent
{_esc(market.get('agent'))}
Model
{_esc(market.get('model'))}
Tokens
{_esc(market.get('tokens'))}
F1
{_esc(market_run_f1_str)}
Answer
{_esc(market.get('answer'))}

Single-agent baseline

Model
{_esc(baseline.get('model'))}
Tokens
{_esc(baseline.get('tokens'))}
F1
{_esc(baseline_run_f1_str)}
Answer
{_esc(baseline.get('answer'))}

Reputation ledger (post-run)

{rep_table}
AgentDomainRunsCorrect TokensScore

Generated by benchmark/report.py. Marketplace primitives are currently stubbed in-process — see benchmark/marketplace.py for the migration plan to the real 016 SynapBus MCP tools.

""" out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text(html_doc, encoding="utf-8")