#!/usr/bin/env python3 """ SynapBus E2E Test Suite Comprehensive end-to-end tests using Claude-powered agents communicating through SynapBus MCP. Generates a self-contained HTML report. Prerequisites: pip install anthropic httpx Usage: # Auto-start server, run all scenarios, generate report: python tests/e2e/run_tests.py --auto-server # Run against existing server: python tests/e2e/run_tests.py --port 8080 # Run single scenario: python tests/e2e/run_tests.py --auto-server --scenario direct_messaging # Keep server running for UI inspection: python tests/e2e/run_tests.py --auto-server --keep-server # Use a different model: python tests/e2e/run_tests.py --auto-server --model claude-haiku-4-5 """ from __future__ import annotations import argparse import os import signal import sys import time from typing import List, Optional # Ensure the test package is importable sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from e2e.lib.auth import create_anthropic_client from e2e.lib.server import find_free_port, start_server, stop_server, wait_for_server from e2e.lib.report import generate_report from e2e.lib.agent_runner import TestResult # Import scenarios from e2e.scenarios import test_direct_messaging from e2e.scenarios import test_channels from e2e.scenarios import test_task_auction from e2e.scenarios import test_access_control from e2e.scenarios import test_agent_discovery from e2e.scenarios import test_blackboard from e2e.scenarios import test_audit_trail # Scenario registry: (name, module) SCENARIOS = [ ("direct_messaging", test_direct_messaging), ("channels", test_channels), ("task_auction", test_task_auction), ("access_control", test_access_control), ("agent_discovery", test_agent_discovery), ("blackboard", test_blackboard), ("audit_trail", test_audit_trail), ] def print_banner(base_url: str, model: str, scenarios: List[str]) -> None: print() print("=" * 60) print(" SynapBus E2E Test Suite") print("=" * 60) print(" Server: {}".format(base_url)) print(" Model: {}".format(model)) print(" Scenarios: {} ({})".format(len(scenarios), ", ".join(scenarios))) print("=" * 60) def print_summary(results: List[TestResult], total_duration: float, report_path: str) -> None: passed = sum(1 for r in results if r.status == "pass") failed = sum(1 for r in results if r.status == "fail") errored = sum(1 for r in results if r.status == "error") total_input = sum(r.total_input_tokens for r in results) total_output = sum(r.total_output_tokens for r in results) print() print("=" * 60) print(" RESULTS") print("=" * 60) for r in results: icon = {"pass": " OK ", "fail": " FAIL", "error": " ERR "}[r.status] print(" [{}] {} ({:.1f}s)".format(icon, r.name, r.duration)) if r.status != "pass" and r.error: # Print first line of error first_line = r.error.strip().split("\n")[-1] print(" {}".format(first_line[:80])) print() print(" Total: {} passed, {} failed, {} errors".format(passed, failed, errored)) print(" Time: {:.1f}s".format(total_duration)) if total_input > 0 or total_output > 0: print(" Tokens: {} in / {} out".format(total_input, total_output)) # Estimate cost (Sonnet pricing) cost = (total_input * 3 + total_output * 15) / 1_000_000 print(" Cost: ${:.4f}".format(cost)) print(" Report: {}".format(os.path.abspath(report_path))) print("=" * 60) def main() -> int: parser = argparse.ArgumentParser( description="SynapBus E2E Test Suite", formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--port", type=int, default=0, help="SynapBus server port (0 = auto-assign)") parser.add_argument("--auto-server", action="store_true", help="Auto-start and stop SynapBus server") parser.add_argument("--keep-server", action="store_true", help="Keep server running after tests for UI inspection") parser.add_argument("--model", default="claude-sonnet-4-6", help="Claude model to use (default: claude-sonnet-4-6)") parser.add_argument("--scenario", type=str, default=None, help="Run a single scenario by name") parser.add_argument("--report", type=str, default="report.html", help="Output HTML report path (default: report.html)") args = parser.parse_args() server_proc = None data_dir = None # Determine which scenarios to run if args.scenario: selected = [(name, mod) for name, mod in SCENARIOS if name == args.scenario] if not selected: available = [name for name, _ in SCENARIOS] print("ERROR: Unknown scenario '{}'. Available: {}".format( args.scenario, ", ".join(available))) return 1 else: selected = SCENARIOS # Start or connect to server if args.auto_server or args.port == 0: port = args.port if args.port != 0 else find_free_port() print("Starting SynapBus server on port {}...".format(port)) server_proc, data_dir, port = start_server(port) print(" Server started (pid={})".format(server_proc.pid)) else: port = args.port base_url = "http://localhost:{}".format(port) if not wait_for_server(base_url, timeout=5): print("ERROR: Server at {} is not responding".format(base_url)) return 1 base_url = "http://localhost:{}".format(port) try: scenario_names = [name for name, _ in selected] print_banner(base_url, args.model, scenario_names) # Authenticate with Anthropic print("\n Authenticating with Anthropic...") claude = create_anthropic_client() # Run scenarios results: List[TestResult] = [] total_start = time.time() for i, (name, module) in enumerate(selected): print("\n--- [{}/{}] {} ---".format(i + 1, len(selected), name)) result = module.run(claude, base_url, args.model) results.append(result) status_str = {"pass": "PASS", "fail": "FAIL", "error": "ERROR"}[result.status] print(" => {} ({:.1f}s)".format(status_str, result.duration)) total_duration = time.time() - total_start # Generate report report_path = generate_report( results=results, model=args.model, base_url=base_url, total_duration=total_duration, output_path=args.report, ) print_summary(results, total_duration, report_path) # Keep server running if requested if args.keep_server and server_proc: print("\n Server running at {} -- press Ctrl+C to stop".format(base_url)) try: signal.pause() except (KeyboardInterrupt, AttributeError): # AttributeError: signal.pause not available on some platforms try: while True: time.sleep(1) except KeyboardInterrupt: pass print("\nStopping server...") # Return exit code if all(r.status == "pass" for r in results): return 0 return 1 finally: if server_proc and not args.keep_server: print("Stopping server...") stop_server(server_proc, data_dir) elif server_proc and args.keep_server: stop_server(server_proc, data_dir) if __name__ == "__main__": sys.exit(main())