277 lines
9.7 KiB
Python
277 lines
9.7 KiB
Python
"""
|
|
Agent Replay Promotion Gate
|
|
===========================
|
|
|
|
Final offline gate before an OpenClaw replacement candidate can move toward
|
|
production shadow/canary. This gate joins the contract report, scorecard, and
|
|
raw candidate metadata so contract probes cannot be mistaken for real evidence.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from src.services.agent_replacement_evaluator import BASELINE_CANDIDATE_ID
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AgentReplayPromotionGateReport:
|
|
"""Promotion decision for one candidate and one target stage."""
|
|
|
|
candidate_id: str
|
|
target_stage: str
|
|
approved: bool
|
|
decision: str
|
|
failures: list[str] = field(default_factory=list)
|
|
evidence: dict[str, Any] = field(default_factory=dict)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "agent_replay_promotion_gate_v1",
|
|
"candidate_id": self.candidate_id,
|
|
"target_stage": self.target_stage,
|
|
"approved": self.approved,
|
|
"decision": self.decision,
|
|
"failures": list(self.failures),
|
|
"evidence": dict(self.evidence),
|
|
}
|
|
|
|
|
|
def evaluate_agent_replay_promotion_gate(
|
|
*,
|
|
candidate_id: str,
|
|
scorecard_report: dict[str, Any],
|
|
contract_report: dict[str, Any],
|
|
raw_results: list[dict[str, Any]],
|
|
import_report: dict[str, Any] | None = None,
|
|
target_stage: str = "shadow",
|
|
) -> AgentReplayPromotionGateReport:
|
|
"""Evaluate whether one candidate may move past offline replay."""
|
|
failures: list[str] = []
|
|
candidate_scorecard = _find_candidate_scorecard(scorecard_report, candidate_id)
|
|
if candidate_id == BASELINE_CANDIDATE_ID:
|
|
failures.append("baseline_candidate_not_promotable")
|
|
|
|
_evaluate_contract(candidate_id, contract_report, failures)
|
|
_evaluate_raw_results(candidate_id, raw_results, failures)
|
|
_evaluate_import_report(
|
|
candidate_id,
|
|
import_report,
|
|
contract_report,
|
|
raw_results,
|
|
failures,
|
|
)
|
|
_evaluate_scorecard(candidate_scorecard, failures)
|
|
|
|
approved = not failures
|
|
return AgentReplayPromotionGateReport(
|
|
candidate_id=candidate_id,
|
|
target_stage=target_stage,
|
|
approved=approved,
|
|
decision="approved" if approved else "blocked",
|
|
failures=failures,
|
|
evidence=_evidence(
|
|
candidate_scorecard=candidate_scorecard,
|
|
contract_report=contract_report,
|
|
raw_results=raw_results,
|
|
import_report=import_report,
|
|
),
|
|
)
|
|
|
|
|
|
def _evaluate_contract(
|
|
candidate_id: str,
|
|
contract_report: dict[str, Any],
|
|
failures: list[str],
|
|
) -> None:
|
|
if contract_report.get("valid") is not True:
|
|
failures.append("contract_invalid")
|
|
if contract_report.get("candidate_id") != candidate_id:
|
|
failures.append(
|
|
"contract_candidate_mismatch:"
|
|
f"expected={candidate_id};actual={contract_report.get('candidate_id')}"
|
|
)
|
|
|
|
|
|
def _evaluate_raw_results(
|
|
candidate_id: str,
|
|
raw_results: list[dict[str, Any]],
|
|
failures: list[str],
|
|
) -> None:
|
|
if not raw_results:
|
|
failures.append("raw_results_empty")
|
|
return
|
|
|
|
raw_candidate_ids = {
|
|
str(result.get("candidate_id", "")).strip()
|
|
for result in raw_results
|
|
if str(result.get("candidate_id", "")).strip()
|
|
}
|
|
if raw_candidate_ids != {candidate_id}:
|
|
failures.append(
|
|
"raw_candidate_mismatch:"
|
|
f"expected={candidate_id};actual={','.join(sorted(raw_candidate_ids))}"
|
|
)
|
|
|
|
not_evidence = [
|
|
result
|
|
for result in raw_results
|
|
if bool((result.get("metadata") or {}).get("not_replacement_evidence"))
|
|
]
|
|
if not_evidence:
|
|
failures.append(f"not_replacement_evidence_present:{len(not_evidence)}")
|
|
|
|
probes = [
|
|
result
|
|
for result in raw_results
|
|
if (result.get("metadata") or {}).get("adapter_mode") == "contract_probe"
|
|
]
|
|
if probes:
|
|
failures.append(f"contract_probe_result_present:{len(probes)}")
|
|
|
|
errors = [result for result in raw_results if result.get("error")]
|
|
if errors:
|
|
failures.append(f"candidate_result_errors_present:{len(errors)}")
|
|
|
|
|
|
def _evaluate_scorecard(
|
|
candidate_scorecard: dict[str, Any] | None,
|
|
failures: list[str],
|
|
) -> None:
|
|
if candidate_scorecard is None:
|
|
failures.append("scorecard_candidate_missing")
|
|
return
|
|
|
|
if candidate_scorecard.get("hard_gates_pass") is not True:
|
|
failures.append("scorecard_hard_gates_failed")
|
|
if candidate_scorecard.get("eligible_for_canary") is not True:
|
|
failures.append("scorecard_not_eligible_for_canary")
|
|
if candidate_scorecard.get("beats_baseline") is not True:
|
|
failures.append("candidate_does_not_beat_baseline")
|
|
|
|
for failure in candidate_scorecard.get("gate_failures") or []:
|
|
if str(failure).startswith("sample_too_small:"):
|
|
failures.append(str(failure))
|
|
|
|
|
|
def _evaluate_import_report(
|
|
candidate_id: str,
|
|
import_report: dict[str, Any] | None,
|
|
contract_report: dict[str, Any],
|
|
raw_results: list[dict[str, Any]],
|
|
failures: list[str],
|
|
) -> None:
|
|
if candidate_id == "nemo_nemotron_fabric" and import_report is None:
|
|
failures.append("nemotron_import_report_missing")
|
|
return
|
|
if import_report is None:
|
|
return
|
|
|
|
if import_report.get("valid") is not True:
|
|
failures.append("import_report_invalid")
|
|
if import_report.get("candidate_id") != candidate_id:
|
|
failures.append(
|
|
"import_report_candidate_mismatch:"
|
|
f"expected={candidate_id};actual={import_report.get('candidate_id')}"
|
|
)
|
|
|
|
imported_results = int(import_report.get("imported_results") or 0)
|
|
if imported_results != len(raw_results):
|
|
failures.append(
|
|
"import_report_raw_result_count_mismatch:"
|
|
f"imported={imported_results};raw={len(raw_results)}"
|
|
)
|
|
|
|
contract_results = int(contract_report.get("results") or 0)
|
|
if contract_results and imported_results != contract_results:
|
|
failures.append(
|
|
"import_report_contract_result_count_mismatch:"
|
|
f"imported={imported_results};contract={contract_results}"
|
|
)
|
|
|
|
requests = import_report.get("requests")
|
|
contract_inputs = int(contract_report.get("inputs") or 0)
|
|
if requests is not None and contract_inputs and int(requests) != contract_inputs:
|
|
failures.append(
|
|
"import_report_contract_input_count_mismatch:"
|
|
f"requests={requests};contract={contract_inputs}"
|
|
)
|
|
|
|
for key in ("duplicate_results", "missing_results", "unexpected_results"):
|
|
values = list(import_report.get(key) or [])
|
|
if values:
|
|
failures.append(f"import_report_{key}_present:{len(values)}")
|
|
|
|
external_errors = int(import_report.get("external_error_records") or 0)
|
|
if external_errors:
|
|
failures.append(f"import_report_external_errors_present:{external_errors}")
|
|
|
|
|
|
def _find_candidate_scorecard(
|
|
scorecard_report: dict[str, Any],
|
|
candidate_id: str,
|
|
) -> dict[str, Any] | None:
|
|
for candidate in scorecard_report.get("candidates") or []:
|
|
if candidate.get("candidate_id") == candidate_id:
|
|
return dict(candidate)
|
|
return None
|
|
|
|
|
|
def _evidence(
|
|
*,
|
|
candidate_scorecard: dict[str, Any] | None,
|
|
contract_report: dict[str, Any],
|
|
raw_results: list[dict[str, Any]],
|
|
import_report: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
metadata = [dict(result.get("metadata") or {}) for result in raw_results]
|
|
return {
|
|
"contract_valid": bool(contract_report.get("valid")),
|
|
"contract_inputs": int(contract_report.get("inputs") or 0),
|
|
"contract_results": int(contract_report.get("results") or 0),
|
|
"raw_results": len(raw_results),
|
|
"not_replacement_evidence_records": sum(
|
|
1 for item in metadata if item.get("not_replacement_evidence")
|
|
),
|
|
"contract_probe_records": sum(
|
|
1 for item in metadata if item.get("adapter_mode") == "contract_probe"
|
|
),
|
|
"candidate_result_error_records": sum(
|
|
1 for result in raw_results if result.get("error")
|
|
),
|
|
"import_report": _import_report_evidence(import_report),
|
|
"scorecard": _scorecard_evidence(candidate_scorecard),
|
|
}
|
|
|
|
|
|
def _scorecard_evidence(candidate_scorecard: dict[str, Any] | None) -> dict[str, Any]:
|
|
if candidate_scorecard is None:
|
|
return {}
|
|
return {
|
|
"incidents": candidate_scorecard.get("incidents"),
|
|
"total_score": candidate_scorecard.get("total_score"),
|
|
"hard_gates_pass": candidate_scorecard.get("hard_gates_pass"),
|
|
"eligible_for_canary": candidate_scorecard.get("eligible_for_canary"),
|
|
"beats_baseline": candidate_scorecard.get("beats_baseline"),
|
|
"gate_failures": list(candidate_scorecard.get("gate_failures") or []),
|
|
}
|
|
|
|
|
|
def _import_report_evidence(import_report: dict[str, Any] | None) -> dict[str, Any]:
|
|
if import_report is None:
|
|
return {"provided": False}
|
|
return {
|
|
"provided": True,
|
|
"valid": import_report.get("valid"),
|
|
"external_results": import_report.get("external_results"),
|
|
"imported_results": import_report.get("imported_results"),
|
|
"requests": import_report.get("requests"),
|
|
"external_error_records": import_report.get("external_error_records"),
|
|
"fallback_used_records": import_report.get("fallback_used_records"),
|
|
"incomplete_trace_records": import_report.get("incomplete_trace_records"),
|
|
"total_cost_usd": import_report.get("total_cost_usd"),
|
|
"avg_latency_ms": import_report.get("avg_latency_ms"),
|
|
"p95_latency_ms": import_report.get("p95_latency_ms"),
|
|
}
|