""" Agent Replay Promotion Gate =========================== Final offline gate before an OpenClaw replacement candidate can move toward production shadow/canary. This gate joins the contract report, scorecard, and raw candidate metadata so contract probes cannot be mistaken for real evidence. """ from __future__ import annotations from dataclasses import dataclass, field from typing import Any from src.services.agent_replacement_evaluator import BASELINE_CANDIDATE_ID @dataclass(frozen=True) class AgentReplayPromotionGateReport: """Promotion decision for one candidate and one target stage.""" candidate_id: str target_stage: str approved: bool decision: str failures: list[str] = field(default_factory=list) evidence: dict[str, Any] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: return { "schema_version": "agent_replay_promotion_gate_v1", "candidate_id": self.candidate_id, "target_stage": self.target_stage, "approved": self.approved, "decision": self.decision, "failures": list(self.failures), "evidence": dict(self.evidence), } def evaluate_agent_replay_promotion_gate( *, candidate_id: str, scorecard_report: dict[str, Any], contract_report: dict[str, Any], raw_results: list[dict[str, Any]], import_report: dict[str, Any] | None = None, target_stage: str = "shadow", ) -> AgentReplayPromotionGateReport: """Evaluate whether one candidate may move past offline replay.""" failures: list[str] = [] candidate_scorecard = _find_candidate_scorecard(scorecard_report, candidate_id) if candidate_id == BASELINE_CANDIDATE_ID: failures.append("baseline_candidate_not_promotable") _evaluate_contract(candidate_id, contract_report, failures) _evaluate_raw_results(candidate_id, raw_results, failures) _evaluate_import_report( candidate_id, import_report, contract_report, raw_results, failures, ) _evaluate_scorecard(candidate_scorecard, failures) approved = not failures return AgentReplayPromotionGateReport( candidate_id=candidate_id, target_stage=target_stage, approved=approved, decision="approved" if approved else "blocked", failures=failures, evidence=_evidence( candidate_scorecard=candidate_scorecard, contract_report=contract_report, raw_results=raw_results, import_report=import_report, ), ) def _evaluate_contract( candidate_id: str, contract_report: dict[str, Any], failures: list[str], ) -> None: if contract_report.get("valid") is not True: failures.append("contract_invalid") if contract_report.get("candidate_id") != candidate_id: failures.append( "contract_candidate_mismatch:" f"expected={candidate_id};actual={contract_report.get('candidate_id')}" ) def _evaluate_raw_results( candidate_id: str, raw_results: list[dict[str, Any]], failures: list[str], ) -> None: if not raw_results: failures.append("raw_results_empty") return raw_candidate_ids = { str(result.get("candidate_id", "")).strip() for result in raw_results if str(result.get("candidate_id", "")).strip() } if raw_candidate_ids != {candidate_id}: failures.append( "raw_candidate_mismatch:" f"expected={candidate_id};actual={','.join(sorted(raw_candidate_ids))}" ) not_evidence = [ result for result in raw_results if bool((result.get("metadata") or {}).get("not_replacement_evidence")) ] if not_evidence: failures.append(f"not_replacement_evidence_present:{len(not_evidence)}") probes = [ result for result in raw_results if (result.get("metadata") or {}).get("adapter_mode") == "contract_probe" ] if probes: failures.append(f"contract_probe_result_present:{len(probes)}") errors = [result for result in raw_results if result.get("error")] if errors: failures.append(f"candidate_result_errors_present:{len(errors)}") def _evaluate_scorecard( candidate_scorecard: dict[str, Any] | None, failures: list[str], ) -> None: if candidate_scorecard is None: failures.append("scorecard_candidate_missing") return if candidate_scorecard.get("hard_gates_pass") is not True: failures.append("scorecard_hard_gates_failed") if candidate_scorecard.get("eligible_for_canary") is not True: failures.append("scorecard_not_eligible_for_canary") if candidate_scorecard.get("beats_baseline") is not True: failures.append("candidate_does_not_beat_baseline") for failure in candidate_scorecard.get("gate_failures") or []: if str(failure).startswith("sample_too_small:"): failures.append(str(failure)) def _evaluate_import_report( candidate_id: str, import_report: dict[str, Any] | None, contract_report: dict[str, Any], raw_results: list[dict[str, Any]], failures: list[str], ) -> None: if candidate_id == "nemo_nemotron_fabric" and import_report is None: failures.append("nemotron_import_report_missing") return if import_report is None: return if import_report.get("valid") is not True: failures.append("import_report_invalid") if import_report.get("candidate_id") != candidate_id: failures.append( "import_report_candidate_mismatch:" f"expected={candidate_id};actual={import_report.get('candidate_id')}" ) imported_results = int(import_report.get("imported_results") or 0) if imported_results != len(raw_results): failures.append( "import_report_raw_result_count_mismatch:" f"imported={imported_results};raw={len(raw_results)}" ) contract_results = int(contract_report.get("results") or 0) if contract_results and imported_results != contract_results: failures.append( "import_report_contract_result_count_mismatch:" f"imported={imported_results};contract={contract_results}" ) requests = import_report.get("requests") contract_inputs = int(contract_report.get("inputs") or 0) if requests is not None and contract_inputs and int(requests) != contract_inputs: failures.append( "import_report_contract_input_count_mismatch:" f"requests={requests};contract={contract_inputs}" ) for key in ("duplicate_results", "missing_results", "unexpected_results"): values = list(import_report.get(key) or []) if values: failures.append(f"import_report_{key}_present:{len(values)}") external_errors = int(import_report.get("external_error_records") or 0) if external_errors: failures.append(f"import_report_external_errors_present:{external_errors}") def _find_candidate_scorecard( scorecard_report: dict[str, Any], candidate_id: str, ) -> dict[str, Any] | None: for candidate in scorecard_report.get("candidates") or []: if candidate.get("candidate_id") == candidate_id: return dict(candidate) return None def _evidence( *, candidate_scorecard: dict[str, Any] | None, contract_report: dict[str, Any], raw_results: list[dict[str, Any]], import_report: dict[str, Any] | None = None, ) -> dict[str, Any]: metadata = [dict(result.get("metadata") or {}) for result in raw_results] return { "contract_valid": bool(contract_report.get("valid")), "contract_inputs": int(contract_report.get("inputs") or 0), "contract_results": int(contract_report.get("results") or 0), "raw_results": len(raw_results), "not_replacement_evidence_records": sum( 1 for item in metadata if item.get("not_replacement_evidence") ), "contract_probe_records": sum( 1 for item in metadata if item.get("adapter_mode") == "contract_probe" ), "candidate_result_error_records": sum( 1 for result in raw_results if result.get("error") ), "import_report": _import_report_evidence(import_report), "scorecard": _scorecard_evidence(candidate_scorecard), } def _scorecard_evidence(candidate_scorecard: dict[str, Any] | None) -> dict[str, Any]: if candidate_scorecard is None: return {} return { "incidents": candidate_scorecard.get("incidents"), "total_score": candidate_scorecard.get("total_score"), "hard_gates_pass": candidate_scorecard.get("hard_gates_pass"), "eligible_for_canary": candidate_scorecard.get("eligible_for_canary"), "beats_baseline": candidate_scorecard.get("beats_baseline"), "gate_failures": list(candidate_scorecard.get("gate_failures") or []), } def _import_report_evidence(import_report: dict[str, Any] | None) -> dict[str, Any]: if import_report is None: return {"provided": False} return { "provided": True, "valid": import_report.get("valid"), "external_results": import_report.get("external_results"), "imported_results": import_report.get("imported_results"), "requests": import_report.get("requests"), "external_error_records": import_report.get("external_error_records"), "fallback_used_records": import_report.get("fallback_used_records"), "incomplete_trace_records": import_report.get("incomplete_trace_records"), "total_cost_usd": import_report.get("total_cost_usd"), "avg_latency_ms": import_report.get("avg_latency_ms"), "p95_latency_ms": import_report.get("p95_latency_ms"), }