feat(governance): 新增 matched PlayBook 學習缺口
All checks were successful
Code Review / ai-code-review (push) Successful in 13s
CD Pipeline / tests (push) Successful in 1m24s
CD Pipeline / build-and-deploy (push) Successful in 4m48s
CD Pipeline / post-deploy-checks (push) Successful in 1m49s

This commit is contained in:
Your Name
2026-06-13 01:05:48 +08:00
parent f9619b137e
commit 3928e3ae67
14 changed files with 1548 additions and 9 deletions

View File

@@ -73,6 +73,9 @@ from src.services.ai_agent_learning_writeback_approval_package import (
from src.services.ai_agent_live_read_model_gate import (
load_latest_ai_agent_live_read_model_gate,
)
from src.services.ai_agent_matched_playbook_learning_gap import (
load_latest_ai_agent_matched_playbook_learning_gap,
)
from src.services.ai_agent_owner_approved_fixture_dry_run import (
load_latest_ai_agent_owner_approved_fixture_dry_run,
)
@@ -1128,6 +1131,34 @@ async def get_agent_task_result_audit_trail() -> dict[str, Any]:
) from exc
@router.get(
"/agent-matched-playbook-learning-gap",
response_model=dict[str, Any],
summary="取得 AI Agent matched PlayBook 學習缺口",
description=(
"讀取最新已提交的 P2-104 matched_playbook_id 學習缺口契約;此端點只回傳 "
"PlayBook id 來源、approval record 序列化、learning service、await learning、測試契約、"
"live null-rate gate 與 owner gate不查 live DB、不更新 PlayBook trust、不寫 KM、不寫 timeline、"
"不寫 Gateway queue、不送 Telegram、不呼叫 Bot API、不寫 production target、不讀 secret。"
),
)
async def get_agent_matched_playbook_learning_gap() -> dict[str, Any]:
"""Return the latest read-only AI Agent matched PlayBook learning gap contract."""
try:
return await asyncio.to_thread(load_latest_ai_agent_matched_playbook_learning_gap)
except FileNotFoundError as exc:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=str(exc),
) from exc
except (json.JSONDecodeError, ValueError) as exc:
logger.error("ai_agent_matched_playbook_learning_gap_invalid", error=str(exc))
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="AI Agent matched PlayBook 學習缺口無效",
) from exc
@router.get(
"/agent-owner-approved-fixture-dry-run",
response_model=dict[str, Any],

View File

@@ -0,0 +1,273 @@
"""
AI Agent matched PlayBook learning gap snapshot.
Loads the latest committed P2-104 matched_playbook_id learning gap contract.
This module validates repo-committed evidence only; it never reads live DB rows,
updates PlayBook trust, writes KM, writes timelines, writes Gateway queues,
sends Telegram messages, reads secrets, or starts runtime work.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from src.services.snapshot_paths import default_evaluations_dir
_DEFAULT_EVALUATIONS_DIR = default_evaluations_dir(Path(__file__))
_SNAPSHOT_PATTERN = "ai_agent_matched_playbook_learning_gap_*.json"
_SCHEMA_VERSION = "ai_agent_matched_playbook_learning_gap_v1"
_RUNTIME_AUTHORITY = "matched_playbook_learning_gap_contract_only_no_live_trust_write"
def load_latest_ai_agent_matched_playbook_learning_gap(
evaluations_dir: Path | None = None,
) -> dict[str, Any]:
"""Load the newest committed matched PlayBook learning gap contract."""
directory = evaluations_dir or _DEFAULT_EVALUATIONS_DIR
candidates = sorted(directory.glob(_SNAPSHOT_PATTERN))
if not candidates:
raise FileNotFoundError(f"no AI Agent matched PlayBook learning gap snapshots found in {directory}")
latest = candidates[-1]
with latest.open(encoding="utf-8") as handle:
payload = json.load(handle)
if not isinstance(payload, dict):
raise ValueError(f"{latest}: expected JSON object")
_require_schema(payload, str(latest))
_require_no_live_boundaries(payload, str(latest))
_require_learning_checkpoints(payload, str(latest))
_require_gap_items(payload, str(latest))
_require_test_contracts(payload, str(latest))
_require_owner_gates(payload, str(latest))
_require_redaction_contract(payload, str(latest))
_require_rollup_consistency(payload, str(latest))
return payload
def _require_schema(payload: dict[str, Any], label: str) -> None:
if payload.get("schema_version") != _SCHEMA_VERSION:
raise ValueError(f"{label}: expected schema_version={_SCHEMA_VERSION}")
status = payload.get("program_status") or {}
if status.get("read_only_mode") is not True:
raise ValueError(f"{label}: program_status.read_only_mode must be true")
if status.get("runtime_authority") != _RUNTIME_AUTHORITY:
raise ValueError(f"{label}: runtime_authority must remain {_RUNTIME_AUTHORITY}")
if status.get("current_task_id") != "P2-104":
raise ValueError(f"{label}: current_task_id must be P2-104")
if status.get("next_task_id") != "P2-105":
raise ValueError(f"{label}: next_task_id must be P2-105")
def _require_no_live_boundaries(payload: dict[str, Any], label: str) -> None:
truth = payload.get("learning_gap_truth") or {}
required_true = {
"p2_103_result_audit_loaded",
"matched_playbook_source_contract_ready",
"approval_record_contract_ready",
"learning_service_contract_ready",
"execution_learning_await_contract_ready",
"e2e_test_contract_ready",
"owner_gate_contract_ready",
"all_checkpoints_have_evidence_ref",
}
missing = sorted(field for field in required_true if truth.get(field) is not True)
if missing:
raise ValueError(f"{label}: learning gap readiness flags must remain true: {missing}")
required_false = {
"runtime_execution_enabled",
"live_db_read_enabled",
"playbook_trust_write_enabled",
"km_write_enabled",
"timeline_write_enabled",
"gateway_queue_write_enabled",
"telegram_send_enabled",
"production_write_enabled",
"secret_value_read_enabled",
"host_or_cluster_command_enabled",
"destructive_operation_enabled",
}
unsafe = sorted(field for field in required_false if truth.get(field) is not False)
if unsafe:
raise ValueError(f"{label}: live read/write/send/execution flags must remain false: {unsafe}")
zero_counts = {
"runtime_execution_count_24h",
"live_db_read_count_24h",
"playbook_trust_write_count_24h",
"km_write_count_24h",
"timeline_write_count_24h",
"gateway_queue_write_count_24h",
"telegram_send_count_24h",
"production_write_count_24h",
"secret_value_read_count_24h",
"host_or_cluster_command_count_24h",
"destructive_operation_count_24h",
}
non_zero = sorted(field for field in zero_counts if truth.get(field) != 0)
if non_zero:
raise ValueError(f"{label}: live read/write/send/execution counts must remain zero: {non_zero}")
def _require_learning_checkpoints(payload: dict[str, Any], label: str) -> None:
checkpoints = payload.get("learning_checkpoints") or []
checkpoint_ids = {checkpoint.get("checkpoint_id") for checkpoint in checkpoints}
required = {
"checkpoint_proposal_playbook_match",
"checkpoint_approval_record_serialization",
"checkpoint_learning_service_update",
"checkpoint_execution_await_learning",
"checkpoint_repair_candidate_playbook_id",
"checkpoint_webhook_source_correlation",
"checkpoint_live_null_rate_gate",
}
if checkpoint_ids != required:
raise ValueError(f"{label}: learning checkpoints must match {sorted(required)}")
valid_status = {
"covered_by_test",
"covered_by_existing_tests",
"source_guarded",
"blocked_until_live_evidence",
}
for checkpoint in checkpoints:
checkpoint_id = checkpoint.get("checkpoint_id")
if checkpoint.get("current_status") not in valid_status:
raise ValueError(f"{label}: checkpoint {checkpoint_id} current_status is invalid")
if checkpoint.get("writes_live_state") is not False:
raise ValueError(f"{label}: checkpoint {checkpoint_id} writes_live_state must remain false")
if not isinstance(checkpoint.get("requires_owner_review"), bool):
raise ValueError(f"{label}: checkpoint {checkpoint_id} requires_owner_review must be boolean")
for field in {"source_component", "evidence_ref", "expected_signal", "gap_if_missing"}:
if not checkpoint.get(field):
raise ValueError(f"{label}: checkpoint {checkpoint_id} must list {field}")
if not _is_redacted_sha256(checkpoint.get("evidence_hash")):
raise ValueError(f"{label}: checkpoint {checkpoint_id} must expose evidence_hash")
def _require_gap_items(payload: dict[str, Any], label: str) -> None:
gaps = payload.get("gap_items") or []
if not gaps:
raise ValueError(f"{label}: gap_items must not be empty")
for gap in gaps:
gap_id = gap.get("gap_id")
if gap.get("severity") not in {"critical", "high", "medium", "low"}:
raise ValueError(f"{label}: gap {gap_id} severity is invalid")
if not isinstance(gap.get("blocks_trust_write"), bool):
raise ValueError(f"{label}: gap {gap_id} blocks_trust_write must be boolean")
for field in {"display_name", "current_state", "required_evidence", "owner_next_action"}:
if not gap.get(field):
raise ValueError(f"{label}: gap {gap_id} must list {field}")
def _require_test_contracts(payload: dict[str, Any], label: str) -> None:
tests = payload.get("test_contracts") or []
if len(tests) < 5:
raise ValueError(f"{label}: test_contracts must include at least five tests")
for test in tests:
test_id = test.get("test_id")
if test.get("live_execution_required") is not False:
raise ValueError(f"{label}: test {test_id} live_execution_required must remain false")
if not test.get("test_ref") or "::" not in test.get("test_ref", ""):
raise ValueError(f"{label}: test {test_id} must expose test_ref")
if not test.get("expected_result"):
raise ValueError(f"{label}: test {test_id} must list expected_result")
def _require_owner_gates(payload: dict[str, Any], label: str) -> None:
gates = payload.get("owner_gates") or []
gate_ids = {gate.get("gate_id") for gate in gates}
required = {
"gate_read_only_live_null_rate",
"gate_playbook_trust_write",
"gate_km_timeline_write",
}
if gate_ids != required:
raise ValueError(f"{label}: owner gates must match {sorted(required)}")
for gate in gates:
gate_id = gate.get("gate_id")
if gate.get("approval_required") is not True:
raise ValueError(f"{label}: gate {gate_id} approval_required must remain true")
if gate.get("runtime_write_allowed") is not False:
raise ValueError(f"{label}: gate {gate_id} runtime_write_allowed must remain false")
if not gate.get("blocked_reason"):
raise ValueError(f"{label}: gate {gate_id} must list blocked_reason")
def _require_redaction_contract(payload: dict[str, Any], label: str) -> None:
contract = payload.get("display_redaction_contract") or {}
required_false = {
"raw_prompt_display_allowed",
"private_reasoning_display_allowed",
"secret_value_display_allowed",
"raw_telegram_payload_display_allowed",
"work_window_transcript_display_allowed",
}
if contract.get("redaction_required") is not True:
raise ValueError(f"{label}: redaction_required must be true")
unsafe = sorted(field for field in required_false if contract.get(field) is not False)
if unsafe:
raise ValueError(f"{label}: redaction flags must remain false: {unsafe}")
if not contract.get("allowed_display_fields"):
raise ValueError(f"{label}: allowed_display_fields must not be empty")
if not contract.get("blocked_display_fields"):
raise ValueError(f"{label}: blocked_display_fields must not be empty")
def _require_rollup_consistency(payload: dict[str, Any], label: str) -> None:
rollups = payload.get("rollups") or {}
checkpoints = payload.get("learning_checkpoints") or []
gaps = payload.get("gap_items") or []
tests = payload.get("test_contracts") or []
gates = payload.get("owner_gates") or []
expected = {
"learning_checkpoint_count": len(checkpoints),
"covered_by_test_count": sum(
1
for item in checkpoints
if item.get("current_status") in {"covered_by_test", "covered_by_existing_tests"}
),
"source_guarded_count": sum(1 for item in checkpoints if item.get("current_status") == "source_guarded"),
"blocked_until_live_evidence_count": sum(1 for item in checkpoints if item.get("current_status") == "blocked_until_live_evidence"),
"gap_item_count": len(gaps),
"high_severity_gap_count": sum(1 for item in gaps if item.get("severity") == "high"),
"test_contract_count": len(tests),
"owner_gate_count": len(gates),
"approval_required_gate_count": sum(1 for item in gates if item.get("approval_required") is True),
}
mismatches = {
key: {"expected": value, "actual": rollups.get(key)}
for key, value in expected.items()
if rollups.get(key) != value
}
if mismatches:
raise ValueError(f"{label}: rollup counts mismatch: {mismatches}")
zero_rollups = {
"runtime_execution_count",
"live_db_read_count",
"playbook_trust_write_count",
"km_write_count",
"timeline_write_count",
"gateway_queue_write_count",
"telegram_send_count",
"production_write_count",
"secret_value_read_count",
"destructive_operation_count",
}
non_zero = sorted(field for field in zero_rollups if rollups.get(field) != 0)
if non_zero:
raise ValueError(f"{label}: rollup live read/write/send/execution counts must remain zero: {non_zero}")
def _is_redacted_sha256(value: Any) -> bool:
if not isinstance(value, str):
return False
if not value.startswith("sha256:"):
return False
digest = value.removeprefix("sha256:")
return len(digest) == 64 and all(char in "0123456789abcdef" for char in digest)

View File

@@ -0,0 +1,116 @@
import copy
import json
import pytest
from src.services.ai_agent_matched_playbook_learning_gap import (
load_latest_ai_agent_matched_playbook_learning_gap,
)
def _write_snapshot(tmp_path, payload):
path = tmp_path / "ai_agent_matched_playbook_learning_gap_2026-06-13.json"
path.write_text(json.dumps(payload), encoding="utf-8")
return path
def test_load_latest_ai_agent_matched_playbook_learning_gap():
data = load_latest_ai_agent_matched_playbook_learning_gap()
assert data["schema_version"] == "ai_agent_matched_playbook_learning_gap_v1"
assert data["program_status"]["current_task_id"] == "P2-104"
assert data["program_status"]["next_task_id"] == "P2-105"
assert data["program_status"]["overall_completion_percent"] == 100
assert data["learning_gap_truth"]["matched_playbook_source_contract_ready"] is True
assert data["learning_gap_truth"]["approval_record_contract_ready"] is True
assert data["learning_gap_truth"]["learning_service_contract_ready"] is True
assert data["learning_gap_truth"]["runtime_execution_enabled"] is False
assert data["learning_gap_truth"]["live_db_read_enabled"] is False
assert data["learning_gap_truth"]["playbook_trust_write_enabled"] is False
assert data["learning_gap_truth"]["km_write_enabled"] is False
assert data["learning_gap_truth"]["timeline_write_enabled"] is False
assert data["learning_gap_truth"]["gateway_queue_write_enabled"] is False
assert data["learning_gap_truth"]["telegram_send_enabled"] is False
assert data["rollups"]["learning_checkpoint_count"] == 7
assert data["rollups"]["covered_by_test_count"] == 4
assert data["rollups"]["source_guarded_count"] == 2
assert data["rollups"]["blocked_until_live_evidence_count"] == 1
assert data["rollups"]["gap_item_count"] == 4
assert data["rollups"]["high_severity_gap_count"] == 2
assert data["rollups"]["test_contract_count"] == 5
assert data["rollups"]["owner_gate_count"] == 3
assert data["rollups"]["approval_required_gate_count"] == 3
assert data["rollups"]["live_db_read_count"] == 0
assert data["rollups"]["playbook_trust_write_count"] == 0
assert data["rollups"]["telegram_send_count"] == 0
def test_rejects_live_db_read_enabled(tmp_path):
data = load_latest_ai_agent_matched_playbook_learning_gap()
bad = copy.deepcopy(data)
bad["learning_gap_truth"]["live_db_read_enabled"] = True
_write_snapshot(tmp_path, bad)
with pytest.raises(ValueError, match="live read/write/send/execution flags"):
load_latest_ai_agent_matched_playbook_learning_gap(tmp_path)
def test_rejects_playbook_trust_write_count(tmp_path):
data = load_latest_ai_agent_matched_playbook_learning_gap()
bad = copy.deepcopy(data)
bad["learning_gap_truth"]["playbook_trust_write_count_24h"] = 1
bad["rollups"]["playbook_trust_write_count"] = 1
_write_snapshot(tmp_path, bad)
with pytest.raises(ValueError, match="live read/write/send/execution counts"):
load_latest_ai_agent_matched_playbook_learning_gap(tmp_path)
def test_rejects_checkpoint_live_write(tmp_path):
data = load_latest_ai_agent_matched_playbook_learning_gap()
bad = copy.deepcopy(data)
bad["learning_checkpoints"][0]["writes_live_state"] = True
_write_snapshot(tmp_path, bad)
with pytest.raises(ValueError, match="writes_live_state"):
load_latest_ai_agent_matched_playbook_learning_gap(tmp_path)
def test_rejects_checkpoint_without_evidence_ref(tmp_path):
data = load_latest_ai_agent_matched_playbook_learning_gap()
bad = copy.deepcopy(data)
bad["learning_checkpoints"][0]["evidence_ref"] = ""
_write_snapshot(tmp_path, bad)
with pytest.raises(ValueError, match="evidence_ref"):
load_latest_ai_agent_matched_playbook_learning_gap(tmp_path)
def test_rejects_test_contract_live_execution(tmp_path):
data = load_latest_ai_agent_matched_playbook_learning_gap()
bad = copy.deepcopy(data)
bad["test_contracts"][0]["live_execution_required"] = True
_write_snapshot(tmp_path, bad)
with pytest.raises(ValueError, match="live_execution_required"):
load_latest_ai_agent_matched_playbook_learning_gap(tmp_path)
def test_rejects_owner_gate_runtime_write(tmp_path):
data = load_latest_ai_agent_matched_playbook_learning_gap()
bad = copy.deepcopy(data)
bad["owner_gates"][0]["runtime_write_allowed"] = True
_write_snapshot(tmp_path, bad)
with pytest.raises(ValueError, match="runtime_write_allowed"):
load_latest_ai_agent_matched_playbook_learning_gap(tmp_path)
def test_rejects_rollup_mismatch(tmp_path):
data = load_latest_ai_agent_matched_playbook_learning_gap()
bad = copy.deepcopy(data)
bad["rollups"]["learning_checkpoint_count"] = 999
_write_snapshot(tmp_path, bad)
with pytest.raises(ValueError, match="rollup counts"):
load_latest_ai_agent_matched_playbook_learning_gap(tmp_path)

View File

@@ -0,0 +1,37 @@
from fastapi.testclient import TestClient
from src.main import app
def test_get_ai_agent_matched_playbook_learning_gap_api():
client = TestClient(app)
response = client.get("/api/v1/agents/agent-matched-playbook-learning-gap")
assert response.status_code == 200
data = response.json()
assert data["schema_version"] == "ai_agent_matched_playbook_learning_gap_v1"
assert data["program_status"]["current_task_id"] == "P2-104"
assert data["program_status"]["next_task_id"] == "P2-105"
assert data["program_status"]["overall_completion_percent"] == 100
assert data["learning_gap_truth"]["matched_playbook_source_contract_ready"] is True
assert data["learning_gap_truth"]["approval_record_contract_ready"] is True
assert data["learning_gap_truth"]["learning_service_contract_ready"] is True
assert data["learning_gap_truth"]["runtime_execution_enabled"] is False
assert data["learning_gap_truth"]["live_db_read_enabled"] is False
assert data["learning_gap_truth"]["playbook_trust_write_enabled"] is False
assert data["learning_gap_truth"]["km_write_enabled"] is False
assert data["learning_gap_truth"]["timeline_write_enabled"] is False
assert data["learning_gap_truth"]["gateway_queue_write_enabled"] is False
assert data["learning_gap_truth"]["telegram_send_enabled"] is False
assert data["rollups"]["learning_checkpoint_count"] == 7
assert data["rollups"]["covered_by_test_count"] == 4
assert data["rollups"]["source_guarded_count"] == 2
assert data["rollups"]["blocked_until_live_evidence_count"] == 1
assert data["rollups"]["gap_item_count"] == 4
assert data["rollups"]["high_severity_gap_count"] == 2
assert data["rollups"]["test_contract_count"] == 5
assert data["rollups"]["owner_gate_count"] == 3
assert data["rollups"]["approval_required_gate_count"] == 3
assert data["rollups"]["live_db_read_count"] == 0
assert data["rollups"]["playbook_trust_write_count"] == 0
assert data["rollups"]["telegram_send_count"] == 0