feat(governance): 新增報表 fixture readback 證據包
This commit is contained in:
@@ -91,6 +91,9 @@ from src.services.ai_agent_report_automation_review import (
|
||||
from src.services.ai_agent_report_runtime_dry_run import (
|
||||
load_latest_ai_agent_report_runtime_dry_run,
|
||||
)
|
||||
from src.services.ai_agent_report_runtime_fixture_readback import (
|
||||
load_latest_ai_agent_report_runtime_fixture_readback,
|
||||
)
|
||||
from src.services.ai_agent_report_runtime_readiness import (
|
||||
load_latest_ai_agent_report_runtime_readiness,
|
||||
)
|
||||
@@ -973,6 +976,34 @@ async def get_agent_report_runtime_dry_run() -> dict[str, Any]:
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/agent-report-runtime-fixture-readback",
|
||||
response_model=dict[str, Any],
|
||||
summary="取得 AI Agent 報表 runtime fixture readback 證據包",
|
||||
description=(
|
||||
"讀取最新已提交的 P2-403N fixture smoke、Telegram Gateway queue preview readback "
|
||||
"與 verifier dry-run 證據包;此端點不排程實發、不送 Telegram、不呼叫 Bot API、"
|
||||
"不寫 Gateway queue、不寫讀報回執、不啟動 AI runtime worker、不啟動中低風險 auto worker、"
|
||||
"不執行 verifier live readback、不寫 production target、不讀 secret、不回傳內部對話內容。"
|
||||
),
|
||||
)
|
||||
async def get_agent_report_runtime_fixture_readback() -> dict[str, Any]:
|
||||
"""Return the latest read-only AI Agent fixture readback package."""
|
||||
try:
|
||||
return await asyncio.to_thread(load_latest_ai_agent_report_runtime_fixture_readback)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
logger.error("ai_agent_report_runtime_fixture_readback_invalid", error=str(exc))
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="AI Agent 報表 runtime fixture readback 證據包無效",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/agent-owner-approved-fixture-dry-run",
|
||||
response_model=dict[str, Any],
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
"""
|
||||
AI Agent report runtime fixture readback snapshot.
|
||||
|
||||
Loads the latest committed P2-403N fixture smoke / queue preview readback /
|
||||
verifier dry-run contract. This module only validates repo-committed evidence
|
||||
and never writes Telegram Gateway queues, sends Telegram messages, starts AI
|
||||
workers, runs live verifiers, or reads secrets.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from src.services.snapshot_paths import default_evaluations_dir
|
||||
|
||||
_DEFAULT_EVALUATIONS_DIR = default_evaluations_dir(Path(__file__))
|
||||
_SNAPSHOT_PATTERN = "ai_agent_report_runtime_fixture_readback_*.json"
|
||||
_SCHEMA_VERSION = "ai_agent_report_runtime_fixture_readback_v1"
|
||||
_RUNTIME_AUTHORITY = "fixture_smoke_queue_preview_readback_verifier_dry_run_only_no_live_send_or_write"
|
||||
|
||||
|
||||
def load_latest_ai_agent_report_runtime_fixture_readback(
|
||||
evaluations_dir: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Load the newest committed AI Agent fixture readback snapshot."""
|
||||
directory = evaluations_dir or _DEFAULT_EVALUATIONS_DIR
|
||||
candidates = sorted(directory.glob(_SNAPSHOT_PATTERN))
|
||||
if not candidates:
|
||||
raise FileNotFoundError(f"no AI Agent report runtime fixture readback snapshots found in {directory}")
|
||||
|
||||
latest = candidates[-1]
|
||||
with latest.open(encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"{latest}: expected JSON object")
|
||||
_require_schema(payload, str(latest))
|
||||
_require_no_write_boundaries(payload, str(latest))
|
||||
_require_fixture_contract(payload, str(latest))
|
||||
_require_queue_readback_contract(payload, str(latest))
|
||||
_require_verifier_contract(payload, str(latest))
|
||||
_require_agent_roles(payload, str(latest))
|
||||
_require_redaction_contract(payload, str(latest))
|
||||
_require_rollup_consistency(payload, str(latest))
|
||||
return payload
|
||||
|
||||
|
||||
def _require_schema(payload: dict[str, Any], label: str) -> None:
|
||||
if payload.get("schema_version") != _SCHEMA_VERSION:
|
||||
raise ValueError(f"{label}: expected schema_version={_SCHEMA_VERSION}")
|
||||
status = payload.get("program_status") or {}
|
||||
if status.get("read_only_mode") is not True:
|
||||
raise ValueError(f"{label}: program_status.read_only_mode must be true")
|
||||
if status.get("runtime_authority") != _RUNTIME_AUTHORITY:
|
||||
raise ValueError(f"{label}: runtime_authority must remain {_RUNTIME_AUTHORITY}")
|
||||
if status.get("current_task_id") != "P2-403N":
|
||||
raise ValueError(f"{label}: current_task_id must be P2-403N")
|
||||
if status.get("next_task_id") != "P2-404":
|
||||
raise ValueError(f"{label}: next_task_id must be P2-404")
|
||||
|
||||
|
||||
def _require_no_write_boundaries(payload: dict[str, Any], label: str) -> None:
|
||||
truth = payload.get("fixture_readback_truth") or {}
|
||||
required_true = {
|
||||
"fixture_smoke_package_ready",
|
||||
"report_snapshot_hash_ready",
|
||||
"telegram_queue_preview_readback_ready",
|
||||
"readback_verifier_dry_run_ready",
|
||||
"redaction_assertions_ready",
|
||||
"operator_review_packet_ready",
|
||||
}
|
||||
missing = sorted(field for field in required_true if truth.get(field) is not True)
|
||||
if missing:
|
||||
raise ValueError(f"{label}: fixture readiness flags must remain true: {missing}")
|
||||
|
||||
required_false = {
|
||||
"production_delivery_enabled",
|
||||
"telegram_gateway_queue_write_enabled",
|
||||
"telegram_send_enabled",
|
||||
"telegram_bot_api_call_enabled",
|
||||
"delivery_receipt_write_enabled",
|
||||
"ai_runtime_worker_enabled",
|
||||
"medium_low_auto_worker_enabled",
|
||||
"post_action_verifier_live_readback_enabled",
|
||||
"production_write_enabled",
|
||||
"secret_value_read_enabled",
|
||||
"work_window_transcript_display_allowed",
|
||||
}
|
||||
unsafe = sorted(field for field in required_false if truth.get(field) is not False)
|
||||
if unsafe:
|
||||
raise ValueError(f"{label}: live write/send/runtime flags must remain false: {unsafe}")
|
||||
|
||||
zero_counts = {
|
||||
"live_report_delivery_count_24h",
|
||||
"telegram_gateway_queue_write_count_24h",
|
||||
"telegram_send_count_24h",
|
||||
"telegram_bot_api_call_count_24h",
|
||||
"delivery_receipt_write_count_24h",
|
||||
"ai_runtime_worker_run_count_24h",
|
||||
"medium_low_auto_execution_count_24h",
|
||||
"post_action_verifier_live_readback_count_24h",
|
||||
"production_write_count_24h",
|
||||
}
|
||||
non_zero = sorted(field for field in zero_counts if truth.get(field) != 0)
|
||||
if non_zero:
|
||||
raise ValueError(f"{label}: live write/send/runtime counts must remain zero: {non_zero}")
|
||||
|
||||
|
||||
def _require_fixture_contract(payload: dict[str, Any], label: str) -> None:
|
||||
fixtures = payload.get("fixture_smoke_results") or []
|
||||
fixture_ids = {fixture.get("fixture_id") for fixture in fixtures}
|
||||
required = {
|
||||
"report_run_snapshot_fixture",
|
||||
"telegram_digest_payload_fixture",
|
||||
"queue_preview_redaction_fixture",
|
||||
"receipt_redaction_fixture",
|
||||
"medium_low_noop_fixture",
|
||||
}
|
||||
if fixture_ids != required:
|
||||
raise ValueError(f"{label}: fixture smoke results must match {sorted(required)}")
|
||||
|
||||
for fixture in fixtures:
|
||||
fixture_id = fixture.get("fixture_id")
|
||||
if not _is_redacted_sha256(fixture.get("output_hash")):
|
||||
raise ValueError(f"{label}: fixture {fixture_id} must expose a redacted sha256 output_hash")
|
||||
if fixture.get("writes_production") is not False:
|
||||
raise ValueError(f"{label}: fixture {fixture_id} must not write production")
|
||||
if fixture.get("sends_telegram") is not False:
|
||||
raise ValueError(f"{label}: fixture {fixture_id} must not send Telegram")
|
||||
if fixture.get("reads_secret_value") is not False:
|
||||
raise ValueError(f"{label}: fixture {fixture_id} must not read secret value")
|
||||
if fixture.get("live_execution_count_24h") != 0:
|
||||
raise ValueError(f"{label}: fixture {fixture_id} live_execution_count_24h must remain zero")
|
||||
|
||||
|
||||
def _require_queue_readback_contract(payload: dict[str, Any], label: str) -> None:
|
||||
readbacks = payload.get("queue_preview_readbacks") or []
|
||||
readback_ids = {readback.get("readback_id") for readback in readbacks}
|
||||
if readback_ids != {
|
||||
"daily_report_digest_readback",
|
||||
"weekly_report_digest_readback",
|
||||
"monthly_report_digest_readback",
|
||||
}:
|
||||
raise ValueError(f"{label}: queue preview readbacks must cover daily, weekly, monthly")
|
||||
|
||||
for readback in readbacks:
|
||||
readback_id = readback.get("readback_id")
|
||||
if readback.get("recipient_room") != "AwoooI SRE 戰情室":
|
||||
raise ValueError(f"{label}: readback {readback_id} must target AwoooI SRE 戰情室")
|
||||
if readback.get("secret_ref") != "SRE_GROUP_CHAT_ID":
|
||||
raise ValueError(f"{label}: readback {readback_id} must only reference SRE_GROUP_CHAT_ID")
|
||||
if not _is_redacted_sha256(readback.get("preview_hash")):
|
||||
raise ValueError(f"{label}: readback {readback_id} must expose a redacted sha256 preview_hash")
|
||||
if readback.get("payload_redacted") is not True:
|
||||
raise ValueError(f"{label}: readback {readback_id} payload must remain redacted")
|
||||
if readback.get("gateway_queue_write_enabled") is not False:
|
||||
raise ValueError(f"{label}: readback {readback_id} must not write Gateway queue")
|
||||
if readback.get("telegram_send_enabled") is not False:
|
||||
raise ValueError(f"{label}: readback {readback_id} must not send Telegram")
|
||||
if readback.get("direct_bot_api_allowed") is not False:
|
||||
raise ValueError(f"{label}: readback {readback_id} must not allow direct Bot API")
|
||||
if readback.get("queue_write_count_24h") != 0:
|
||||
raise ValueError(f"{label}: readback {readback_id} queue_write_count_24h must remain zero")
|
||||
|
||||
|
||||
def _require_verifier_contract(payload: dict[str, Any], label: str) -> None:
|
||||
cases = payload.get("verifier_dry_run_cases") or []
|
||||
case_ids = {case.get("case_id") for case in cases}
|
||||
required = {
|
||||
"report_snapshot_verifier_dry_run",
|
||||
"gateway_preview_verifier_dry_run",
|
||||
"receipt_redaction_verifier_dry_run",
|
||||
"medium_low_noop_verifier_dry_run",
|
||||
}
|
||||
if case_ids != required:
|
||||
raise ValueError(f"{label}: verifier dry-run cases must match {sorted(required)}")
|
||||
|
||||
for case in cases:
|
||||
case_id = case.get("case_id")
|
||||
if not _is_redacted_sha256(case.get("evidence_hash")):
|
||||
raise ValueError(f"{label}: verifier case {case_id} must expose a redacted sha256 evidence_hash")
|
||||
if case.get("live_readback_enabled") is not False:
|
||||
raise ValueError(f"{label}: verifier case {case_id} must not run live readback")
|
||||
if case.get("writes_result") is not False:
|
||||
raise ValueError(f"{label}: verifier case {case_id} must not write result")
|
||||
if case.get("requires_secret_value") is not False:
|
||||
raise ValueError(f"{label}: verifier case {case_id} must not require secret value")
|
||||
|
||||
|
||||
def _require_agent_roles(payload: dict[str, Any], label: str) -> None:
|
||||
roles = payload.get("agent_fixture_roles") or []
|
||||
agents = {role.get("agent_id") for role in roles}
|
||||
if agents != {"openclaw", "hermes", "nemotron"}:
|
||||
raise ValueError(f"{label}: fixture roles must include OpenClaw, Hermes, and NemoTron")
|
||||
for role in roles:
|
||||
if role.get("live_action_count_24h") != 0:
|
||||
raise ValueError(f"{label}: agent {role.get('agent_id')} live_action_count_24h must remain zero")
|
||||
|
||||
|
||||
def _require_redaction_contract(payload: dict[str, Any], label: str) -> None:
|
||||
contract = payload.get("display_redaction_contract") or {}
|
||||
required_false = {
|
||||
"raw_report_payload_display_allowed",
|
||||
"raw_telegram_payload_display_allowed",
|
||||
"private_reasoning_display_allowed",
|
||||
"secret_value_display_allowed",
|
||||
"work_window_transcript_display_allowed",
|
||||
}
|
||||
if contract.get("redaction_required") is not True:
|
||||
raise ValueError(f"{label}: display redaction must remain required")
|
||||
unsafe = sorted(field for field in required_false if contract.get(field) is not False)
|
||||
if unsafe:
|
||||
raise ValueError(f"{label}: display redaction fields must remain false: {unsafe}")
|
||||
|
||||
|
||||
def _require_rollup_consistency(payload: dict[str, Any], label: str) -> None:
|
||||
rollups = payload.get("rollups") or {}
|
||||
truth = payload.get("fixture_readback_truth") or {}
|
||||
fixtures = payload.get("fixture_smoke_results") or []
|
||||
readbacks = payload.get("queue_preview_readbacks") or []
|
||||
cases = payload.get("verifier_dry_run_cases") or []
|
||||
roles = payload.get("agent_fixture_roles") or []
|
||||
checkpoints = payload.get("operator_checkpoints") or []
|
||||
|
||||
expected = {
|
||||
"fixture_smoke_count": len(fixtures),
|
||||
"passed_fixture_smoke_count": sum(
|
||||
1
|
||||
for fixture in fixtures
|
||||
if fixture.get("smoke_status") in {"passed_no_write", "passed_redaction"}
|
||||
),
|
||||
"queue_preview_readback_count": len(readbacks),
|
||||
"verifier_dry_run_case_count": len(cases),
|
||||
"agent_role_count": len(roles),
|
||||
"operator_checkpoint_count": len(checkpoints),
|
||||
"live_report_delivery_count": truth.get("live_report_delivery_count_24h"),
|
||||
"telegram_gateway_queue_write_count": truth.get("telegram_gateway_queue_write_count_24h"),
|
||||
"telegram_send_count": truth.get("telegram_send_count_24h"),
|
||||
"telegram_bot_api_call_count": truth.get("telegram_bot_api_call_count_24h"),
|
||||
"delivery_receipt_write_count": truth.get("delivery_receipt_write_count_24h"),
|
||||
"ai_runtime_worker_run_count": truth.get("ai_runtime_worker_run_count_24h"),
|
||||
"medium_low_auto_execution_count": truth.get("medium_low_auto_execution_count_24h"),
|
||||
"post_action_verifier_live_readback_count": truth.get("post_action_verifier_live_readback_count_24h"),
|
||||
"production_write_count": truth.get("production_write_count_24h"),
|
||||
}
|
||||
mismatched = {
|
||||
key: {"expected": value, "actual": rollups.get(key)}
|
||||
for key, value in expected.items()
|
||||
if rollups.get(key) != value
|
||||
}
|
||||
if mismatched:
|
||||
raise ValueError(f"{label}: rollup counts must match payload sections: {mismatched}")
|
||||
|
||||
approval_required = sorted(
|
||||
checkpoint.get("checkpoint_id")
|
||||
for checkpoint in checkpoints
|
||||
if checkpoint.get("approval_required") is True
|
||||
)
|
||||
if sorted(rollups.get("approval_required_checkpoint_ids") or []) != approval_required:
|
||||
raise ValueError(f"{label}: approval_required_checkpoint_ids mismatch")
|
||||
|
||||
|
||||
def _is_redacted_sha256(value: Any) -> bool:
|
||||
if not isinstance(value, str) or not value.startswith("sha256:"):
|
||||
return False
|
||||
digest = value.removeprefix("sha256:")
|
||||
return len(digest) == 64 and all(char in "0123456789abcdef" for char in digest)
|
||||
Reference in New Issue
Block a user