feat: 实现可靠事件与证据 connector (#153)
This commit is contained in:
QiuSW
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
|
||||
import yovision_brain.app.__main__ as cli
|
||||
import yovision_brain.integration.event_export.runtime as export_runtime
|
||||
from yovision_brain.integration.event_export import (
|
||||
EventDeliveryError,
|
||||
HTTPSMachineIdentitySender,
|
||||
)
|
||||
from yovision_brain.integration.machine_identity import (
|
||||
KeyRecord,
|
||||
Registry,
|
||||
ReplayStore,
|
||||
Signer,
|
||||
TransportPolicy,
|
||||
Verifier,
|
||||
bearer_token,
|
||||
)
|
||||
|
||||
FIXTURE = Path(__file__).parents[1] / "fixtures" / "events" / "area.json"
|
||||
|
||||
|
||||
def _policy() -> TransportPolicy:
|
||||
return TransportPolicy(
|
||||
tls_min_version="1.2",
|
||||
verify_certificate=True,
|
||||
verify_hostname=True,
|
||||
connect_timeout_ms=1_000,
|
||||
response_header_timeout_ms=1_000,
|
||||
request_timeout_ms=2_000,
|
||||
max_request_bytes=64 * 1024,
|
||||
)
|
||||
|
||||
|
||||
class _Response:
|
||||
status = 202
|
||||
|
||||
def __init__(self, body: bytes, request_id: str) -> None:
|
||||
self.body = body
|
||||
self.headers = {"X-Request-ID": request_id}
|
||||
self.closed = False
|
||||
|
||||
def read(self, amount: int = -1) -> bytes:
|
||||
return self.body[:amount] if amount >= 0 else self.body
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _VerifyingOpener:
|
||||
def __init__(self, verifier: Verifier) -> None:
|
||||
self.verifier = verifier
|
||||
self.requests = []
|
||||
|
||||
def open(self, request, timeout: float) -> _Response: # noqa: ANN001
|
||||
self.requests.append((request, timeout))
|
||||
body = request.data
|
||||
self.verifier.verify(
|
||||
bearer_token(request.get_header("Authorization")),
|
||||
"yovision-sense",
|
||||
"events:ingest",
|
||||
request.method,
|
||||
"/v1/events",
|
||||
body,
|
||||
)
|
||||
event = json.loads(body)
|
||||
return _Response(json.dumps({
|
||||
"producer_id": event["producer_id"],
|
||||
"source_event_id": event["source_event_id"],
|
||||
"payload_sha256": hashlib.sha256(body).hexdigest(),
|
||||
"disposition": "accepted",
|
||||
}).encode(), request.get_header("X-request-id"))
|
||||
|
||||
|
||||
def test_sender_binds_exact_path_body_and_safe_request_id() -> None:
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
signer = Signer("yv:brain:school-a", "brain-key-0001", private_key, clock=lambda: 100)
|
||||
registry = Registry(
|
||||
[
|
||||
KeyRecord(
|
||||
principal="yv:brain:school-a",
|
||||
key_id="brain-key-0001",
|
||||
public_key=private_key.public_key(),
|
||||
audience="yovision-sense",
|
||||
scopes=frozenset({"events:ingest"}),
|
||||
)
|
||||
]
|
||||
)
|
||||
opener = _VerifyingOpener(Verifier(registry, ReplayStore(), clock=lambda: 100))
|
||||
sender = HTTPSMachineIdentitySender(
|
||||
"https://sense.example:8443", signer, _policy(), opener=opener
|
||||
)
|
||||
|
||||
result = sender.send(
|
||||
b'{"producer_id":"brain-school-a","schema_version":"yovision.event/v1",'
|
||||
b'"source_event_id":"evt-1"}'
|
||||
)
|
||||
|
||||
request, timeout = opener.requests[0]
|
||||
assert request.full_url == "https://sense.example:8443/v1/events"
|
||||
assert request.method == "POST"
|
||||
assert request.get_header("Content-type") == "application/json"
|
||||
assert re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{15,127}", result.request_id)
|
||||
assert request.get_header("X-request-id") == result.request_id
|
||||
assert (result.disposition, timeout) == ("accepted", 2.0)
|
||||
|
||||
|
||||
def test_sender_rejects_plaintext_and_marks_conflict_terminal() -> None:
|
||||
key = Ed25519PrivateKey.generate()
|
||||
signer = Signer("yv:brain:school-a", "brain-key-0001", key)
|
||||
with pytest.raises(ValueError, match="HTTPS origin"):
|
||||
HTTPSMachineIdentitySender("http://sense.example", signer, _policy())
|
||||
|
||||
class ConflictOpener:
|
||||
def open(self, request, timeout: float): # noqa: ANN001, ARG002
|
||||
raise urllib.error.HTTPError(
|
||||
request.full_url,
|
||||
409,
|
||||
"Conflict",
|
||||
{},
|
||||
io.BytesIO(b'{"code":"event_identity_conflict"}'),
|
||||
)
|
||||
|
||||
sender = HTTPSMachineIdentitySender(
|
||||
"https://sense.example", signer, _policy(), opener=ConflictOpener()
|
||||
)
|
||||
with pytest.raises(EventDeliveryError) as caught:
|
||||
sender.send(b"{}")
|
||||
assert (caught.value.code, caught.value.terminal) == (
|
||||
"event_identity_conflict",
|
||||
True,
|
||||
)
|
||||
|
||||
|
||||
def test_disabled_connector_keeps_existing_json_lines_output(tmp_path: Path) -> None:
|
||||
raw = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
||||
raw["event_export"] = {"enabled": False}
|
||||
config = tmp_path / "brain.json"
|
||||
config.write_text(json.dumps(raw), encoding="utf-8")
|
||||
output = tmp_path / "events.jsonl"
|
||||
|
||||
assert cli.main(["--config", str(config), "--output", str(output)]) == 0
|
||||
assert json.loads(output.read_text(encoding="utf-8"))["schema"] == (
|
||||
"brain.internal.event-candidate/v1"
|
||||
)
|
||||
|
||||
|
||||
def test_delivery_failure_is_nonzero_and_not_silently_reported_as_success(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
class FailingSink:
|
||||
def write(self, candidate) -> None: # noqa: ANN001, ARG002
|
||||
raise EventDeliveryError("event_delivery_unavailable", terminal=False)
|
||||
|
||||
monkeypatch.setattr(cli, "build_event_export_sink", lambda raw, base_dir: FailingSink())
|
||||
unused_output = tmp_path / "disabled-json-lines-target"
|
||||
unused_output.write_text("must remain unchanged", encoding="utf-8")
|
||||
result = cli.main(
|
||||
["--config", str(FIXTURE), "--output", str(unused_output)]
|
||||
)
|
||||
|
||||
assert result == 3
|
||||
error = json.loads(capsys.readouterr().err.splitlines()[0])
|
||||
assert error == {"status": "error", "message": "event_delivery_unavailable"}
|
||||
assert unused_output.read_text(encoding="utf-8") == "must remain unchanged"
|
||||
|
||||
|
||||
def test_inline_private_key_material_is_rejected_without_echo(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
raw = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
||||
marker = "INLINE-PRIVATE-MATERIAL-MUST-NOT-LEAK"
|
||||
raw["event_export"] = {
|
||||
"enabled": True,
|
||||
"endpoint": "https://sense.example",
|
||||
"producer_id": "brain-school-a",
|
||||
"site_ref": "site-school-a",
|
||||
"severity": "high",
|
||||
"region_refs": {},
|
||||
"crossing_directions": {},
|
||||
"machine_identity": {
|
||||
"principal": "yv:brain:school-a",
|
||||
"key_id": "brain-key-0001",
|
||||
"private_key_path": "external.pem",
|
||||
"private_key": marker,
|
||||
},
|
||||
"transport": {},
|
||||
}
|
||||
config = tmp_path / "brain.json"
|
||||
config.write_text(json.dumps(raw), encoding="utf-8")
|
||||
|
||||
assert cli.main(["--config", str(config)]) == 3
|
||||
assert marker not in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_enabled_config_loads_machine_key_only_from_external_path(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
key_path = tmp_path / "brain-machine.pem"
|
||||
key_path.write_bytes(
|
||||
Ed25519PrivateKey.generate().private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.PKCS8,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
)
|
||||
captured = {}
|
||||
|
||||
class Sender:
|
||||
def __init__(self, endpoint, signer, policy) -> None: # noqa: ANN001
|
||||
captured.update(endpoint=endpoint, signer=signer, policy=policy)
|
||||
|
||||
monkeypatch.setattr(export_runtime, "HTTPSMachineIdentitySender", Sender)
|
||||
sink = export_runtime.build_event_export_sink(
|
||||
{
|
||||
"enabled": True,
|
||||
"endpoint": "https://sense.example",
|
||||
"producer_id": "brain-school-a",
|
||||
"site_ref": "site-school-a",
|
||||
"severity": "high",
|
||||
"region_refs": {},
|
||||
"crossing_directions": {},
|
||||
"machine_identity": {
|
||||
"principal": "yv:brain:school-a",
|
||||
"key_id": "brain-key-0001",
|
||||
"private_key_path": key_path.name,
|
||||
},
|
||||
"transport": {
|
||||
"tls_min_version": "1.2",
|
||||
"verify_certificate": True,
|
||||
"verify_hostname": True,
|
||||
"connect_timeout_ms": 1_000,
|
||||
"response_header_timeout_ms": 1_000,
|
||||
"request_timeout_ms": 2_000,
|
||||
"max_request_bytes": 64 * 1024,
|
||||
},
|
||||
},
|
||||
base_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert sink is not None
|
||||
assert captured["endpoint"] == "https://sense.example"
|
||||
@@ -0,0 +1,148 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from yovision_brain.events import INTERNAL_EVENT_SCHEMA, InternalEventCandidate
|
||||
from yovision_brain.integration.event_export import EventExportError, canonical_json, export_event, payload_sha256
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
|
||||
|
||||
def _candidate(**changes: object) -> InternalEventCandidate:
|
||||
values: dict[str, object] = {
|
||||
"schema": INTERNAL_EVENT_SCHEMA,
|
||||
"event_id": "evt-area-20260831-0001",
|
||||
"logical_input_id": "camera-east-gate",
|
||||
"event_type": "danger_area_entered",
|
||||
"occurred_at_ns": 1_788_134_401_125_000_000,
|
||||
"rule_id": "rule-east-danger",
|
||||
"rule_version": "3",
|
||||
"model_name": "anonymous-detector",
|
||||
"model_version": "2026.08",
|
||||
"profile_id": "profile-main-stream",
|
||||
"frame_width": 100,
|
||||
"frame_height": 100,
|
||||
"track_id": "track-0042",
|
||||
"observation": {
|
||||
"category": "anonymous_target",
|
||||
"confidence": 0.93,
|
||||
"box": {"left": 12, "top": 20, "right": 31, "bottom": 74},
|
||||
"anchor": {"x": 0.21, "y": 0.74},
|
||||
},
|
||||
"reason": "entered polygon",
|
||||
}
|
||||
values.update(changes)
|
||||
return InternalEventCandidate(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _fixture(relative: str) -> dict[str, object]:
|
||||
return json.loads((ROOT / relative).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_brain_mapper_matches_frozen_producer_fixture_and_jcs_digest() -> None:
|
||||
pending = _fixture("contracts/evidence/v1/examples/pending.json")
|
||||
expected = _fixture("contracts/events/v1/examples/dangerous-area.json")
|
||||
event = export_event(
|
||||
_candidate(),
|
||||
producer_id="brain-school-a",
|
||||
site_ref="site-school-a",
|
||||
severity="high",
|
||||
evidence=[pending],
|
||||
region_ref="region-east-danger",
|
||||
)
|
||||
|
||||
assert event == expected
|
||||
assert payload_sha256(event) == "4cc1e93820195caf713ea675ff33f178c9d4997dd8a81cb61287e9fea0e3d5e1"
|
||||
|
||||
|
||||
def test_retry_mapping_preserves_original_identity_and_payload() -> None:
|
||||
candidate = _candidate()
|
||||
arguments = {
|
||||
"producer_id": "brain-school-a",
|
||||
"site_ref": "site-school-a",
|
||||
"severity": "high",
|
||||
"evidence": [_fixture("contracts/evidence/v1/examples/pending.json")],
|
||||
"region_ref": "region-east-danger",
|
||||
}
|
||||
first = export_event(candidate, **arguments)
|
||||
retry = export_event(candidate, **arguments)
|
||||
|
||||
assert (first["producer_id"], first["source_event_id"]) == (
|
||||
"brain-school-a",
|
||||
candidate.event_id,
|
||||
)
|
||||
assert retry == first
|
||||
assert payload_sha256(retry) == payload_sha256(first)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fixture", ["pending.json", "failed.json"])
|
||||
def test_evidence_degradation_states_are_exported_unchanged(fixture: str) -> None:
|
||||
evidence = _fixture(f"contracts/evidence/v1/examples/{fixture}")
|
||||
candidate = _candidate()
|
||||
if fixture == "failed.json":
|
||||
candidate = _candidate(
|
||||
event_id="evt-line-20260831-0002",
|
||||
logical_input_id="camera-north-corridor",
|
||||
event_type="directional_line_crossed",
|
||||
occurred_at_ns=1_788_134_590_000_000_000,
|
||||
rule_id="rule-north-one-way",
|
||||
rule_version="1",
|
||||
track_id="track-0088",
|
||||
observation={"category": "anonymous_target", "confidence": 0.88},
|
||||
)
|
||||
event = export_event(
|
||||
candidate,
|
||||
producer_id="brain-school-a",
|
||||
site_ref="site-school-a",
|
||||
severity="medium" if fixture == "failed.json" else "high",
|
||||
evidence=[evidence],
|
||||
region_ref="line-north-one-way" if fixture == "failed.json" else "region-east-danger",
|
||||
crossing_direction="b_to_a" if fixture == "failed.json" else None,
|
||||
)
|
||||
|
||||
assert event["evidence"] == [evidence]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsafe",
|
||||
[
|
||||
{"local_path": "D:/captures/frame.jpg"},
|
||||
{"url": "https://example.invalid/signed"},
|
||||
{"token": "not-a-real-token"},
|
||||
],
|
||||
)
|
||||
def test_evidence_rejects_paths_urls_and_sensitive_fields(unsafe: dict[str, object]) -> None:
|
||||
evidence = _fixture("contracts/evidence/v1/examples/pending.json")
|
||||
evidence.update(unsafe)
|
||||
with pytest.raises(EventExportError, match="forbidden"):
|
||||
export_event(
|
||||
_candidate(),
|
||||
producer_id="brain-school-a",
|
||||
site_ref="site-school-a",
|
||||
severity="high",
|
||||
evidence=[evidence],
|
||||
region_ref="region-east-danger",
|
||||
)
|
||||
|
||||
|
||||
def test_failed_evidence_cannot_claim_success_content() -> None:
|
||||
evidence = _fixture("contracts/evidence/v1/examples/failed.json")
|
||||
evidence["content_type"] = "video/mp4"
|
||||
with pytest.raises(EventExportError, match="failed evidence"):
|
||||
export_event(
|
||||
_candidate(),
|
||||
producer_id="brain-school-a",
|
||||
site_ref="site-school-a",
|
||||
severity="high",
|
||||
evidence=[evidence],
|
||||
region_ref="region-east-danger",
|
||||
)
|
||||
|
||||
|
||||
def test_jcs_normalizes_number_lexemes_and_negative_zero() -> None:
|
||||
assert canonical_json({"small": 1e-7, "fixed": 1e20, "zero": -0.0}) == (
|
||||
'{"fixed":100000000000000000000,"small":1e-7,"zero":0}'
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
import pytest
|
||||
|
||||
from yovision_brain.integration.event_export import SQLiteReplayCache
|
||||
from yovision_brain.integration.machine_identity import (
|
||||
KeyRecord,
|
||||
MachineIdentityError,
|
||||
Registry,
|
||||
Signer,
|
||||
Verifier,
|
||||
)
|
||||
|
||||
|
||||
def test_replay_cache_rejects_same_jti_after_connector_restart(tmp_path) -> None:
|
||||
now = 1_800_000_000
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
registry = Registry(
|
||||
[
|
||||
KeyRecord(
|
||||
principal="yv:brain:node-a",
|
||||
key_id="brain-key-0001",
|
||||
public_key=private_key.public_key(),
|
||||
audience="yovision-sense",
|
||||
scopes=frozenset({"events:ingest"}),
|
||||
)
|
||||
]
|
||||
)
|
||||
token = Signer(
|
||||
"yv:brain:node-a",
|
||||
"brain-key-0001",
|
||||
private_key,
|
||||
clock=lambda: now,
|
||||
).mint("yovision-sense", ["events:ingest"], "POST", "/v1/events", b"{}")
|
||||
database = tmp_path / "machine-replay.sqlite3"
|
||||
|
||||
first_process = Verifier(registry, SQLiteReplayCache(database), clock=lambda: now)
|
||||
claims = first_process.verify(
|
||||
token, "yovision-sense", "events:ingest", "POST", "/v1/events", b"{}"
|
||||
)
|
||||
assert claims.iss == "yv:brain:node-a"
|
||||
|
||||
restarted_process = Verifier(registry, SQLiteReplayCache(database), clock=lambda: now)
|
||||
with pytest.raises(MachineIdentityError, match="machine_token_replayed") as caught:
|
||||
restarted_process.verify(
|
||||
token, "yovision-sense", "events:ingest", "POST", "/v1/events", b"{}"
|
||||
)
|
||||
assert caught.value.code == "machine_token_replayed"
|
||||
|
||||
|
||||
def test_replay_cache_atomically_reuses_expired_identity(tmp_path) -> None:
|
||||
cache = SQLiteReplayCache(tmp_path / "machine-replay.sqlite3")
|
||||
assert cache.consume("yv:brain:node-a", "jti-one", expires_at=110, now=100)
|
||||
assert not cache.consume("yv:brain:node-a", "jti-one", expires_at=110, now=101)
|
||||
assert cache.consume("yv:brain:node-a", "jti-one", expires_at=130, now=110)
|
||||
Reference in New Issue
Block a user