From e55840f280a82f4838f05143b6e5743cd22f8950 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 29 Jun 2026 20:43:32 +0800 Subject: [PATCH] feat(api): validate credential escrow evidence refs --- apps/api/src/api/v1/agents.py | 34 +++ ...ential_escrow_evidence_intake_readiness.py | 220 ++++++++++++++++++ ...al_escrow_evidence_intake_readiness_api.py | 93 ++++++++ docs/LOGBOOK.md | 19 ++ ...priority-work-order-readback.snapshot.json | 13 +- 5 files changed, 376 insertions(+), 3 deletions(-) diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index 318c7387d..55994ee56 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -321,6 +321,7 @@ from src.services.backup_restore_drill_approval_package_template import ( ) from src.services.credential_escrow_evidence_intake_readiness import ( load_latest_credential_escrow_evidence_intake_readiness, + validate_credential_escrow_evidence_refs, validate_credential_escrow_evidence_owner_response, ) from src.services.delivery_closure_workbench import ( @@ -3970,6 +3971,39 @@ async def validate_credential_escrow_evidence_owner_response_packet( ) from exc +@router.post( + "/credential-escrow-evidence-intake-readiness/validate-evidence-refs", + response_model=dict[str, Any], + summary="驗證 P0-005 credential escrow 脫敏 evidence refs", + description=( + "接收五個 credential escrow item 的 redacted non-secret evidence refs，" + "自動轉成既有 owner response preflight 結構並回傳 reviewer readiness；" + "此端點不保存 payload、不讀 credential、不寫 escrow marker、不執行 backup / " + "restore / offsite sync、不觸發 workflow、不碰 host/K8s、不讀 raw session/SQLite。" + ), +) +async def validate_credential_escrow_evidence_refs_packet( + evidence_refs_payload: dict[str, Any], +) -> dict[str, Any]: + """Return no-persist validation for simplified P0-005 redacted refs.""" + try: + return await asyncio.to_thread( + validate_credential_escrow_evidence_refs, + evidence_refs_payload, + ) + except FileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except (json.JSONDecodeError, ValueError) as exc: + logger.error("credential_escrow_evidence_refs_invalid", error=str(exc)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="P0-005 credential escrow evidence refs 驗證器無效", + ) from exc + + @router.get( "/backup-notification-policy", response_model=dict[str, Any], diff --git a/apps/api/src/services/credential_escrow_evidence_intake_readiness.py b/apps/api/src/services/credential_escrow_evidence_intake_readiness.py index 47dbbb80a..2495b3883 100644 --- a/apps/api/src/services/credential_escrow_evidence_intake_readiness.py +++ b/apps/api/src/services/credential_escrow_evidence_intake_readiness.py @@ -22,6 +22,9 @@ _DEFAULT_SECURITY_DIR = default_security_dir(Path(__file__)) _OWNER_REQUEST_FILE = "credential-escrow-evidence-owner-request.snapshot.json" _SCHEMA_VERSION = "credential_escrow_evidence_intake_readiness_v1" _VALIDATION_SCHEMA_VERSION = "credential_escrow_evidence_owner_response_validation_v1" +_EVIDENCE_REFS_VALIDATION_SCHEMA_VERSION = ( + "credential_escrow_evidence_refs_validation_v1" +) _SINGLE_PREFLIGHT_INTAKE_SCHEMA_VERSION = "credential_escrow_single_preflight_intake_v1" _OWNER_PACKET_SCHEMA = "awoooi_post_reboot_next_gate_owner_packets_v1" _OWNER_REQUEST_SCOPE = "credential_escrow_evidence_owner_request" @@ -286,6 +289,107 @@ def validate_credential_escrow_evidence_owner_response( } +def validate_credential_escrow_evidence_refs( + evidence_refs_payload: dict[str, Any], + readiness: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Validate simplified redacted evidence refs without persistence or marker writes.""" + current = readiness or load_latest_credential_escrow_evidence_intake_readiness() + source_sensitive_hits = _find_sensitive_strings(evidence_refs_payload) + source_forbidden_boolean_hits = _find_forbidden_booleans(evidence_refs_payload) + owner_response = _owner_response_from_evidence_refs(evidence_refs_payload) + validation = validate_credential_escrow_evidence_owner_response( + owner_response, + readiness=current, + ) + rollups = _dict(current.get("rollups")) + submissions = _normalized_evidence_ref_submissions(evidence_refs_payload) + provided_item_ids = sorted( + { + str(item.get("item_id") or "").strip() + for item in submissions + if str(item.get("item_id") or "").strip() in _REQUIRED_ITEM_IDS + } + ) + + status = str(validation.get("status") or "") + if source_forbidden_boolean_hits: + status = "rejected_runtime_or_marker_action" + elif source_sensitive_hits: + status = "quarantined_sensitive_payload" + + result = dict(_dict(validation.get("result"))) + result.update( + { + "simplified_evidence_ref_submission_count": len(submissions), + "provided_required_item_ids": provided_item_ids, + "missing_required_item_ids": sorted( + _REQUIRED_ITEM_IDS - set(provided_item_ids) + ), + "source_sensitive_payload_hit_count": len(source_sensitive_hits), + "source_forbidden_true_field_count": len(source_forbidden_boolean_hits), + "payload_persisted": False, + "runtime_gate_count": 0, + "credential_marker_write_authorized_count": 0, + "secret_value_collection_allowed": False, + } + ) + if status != validation.get("status"): + result["owner_response_accepted_count"] = 0 + result["projected_effective_escrow_missing_count"] = _int( + rollups.get("effective_escrow_missing_count") + ) + + reviewer_readiness = dict(_dict(validation.get("reviewer_readiness"))) + if status != validation.get("status"): + reviewer_readiness.update( + { + "status": "not_ready_for_reviewer_acceptance_writeback", + "redacted_receipt_writeback_ready_count": 0, + "projected_effective_escrow_missing_count": _int( + rollups.get("effective_escrow_missing_count") + ), + } + ) + + return { + "schema_version": _EVIDENCE_REFS_VALIDATION_SCHEMA_VERSION, + "status": status, + "priority": "P0-005", + "scope": "credential_escrow_evidence_refs_intake", + "source_readiness_status": current.get("status"), + "result": result, + "blockers": validation.get("blockers", []), + "sensitive_payload_hits": sorted( + set(_list(validation.get("sensitive_payload_hits")) + source_sensitive_hits) + ), + "forbidden_boolean_hits": sorted( + set( + _list(validation.get("forbidden_boolean_hits")) + + source_forbidden_boolean_hits + ) + ), + "operation_boundaries": _dict(validation.get("operation_boundaries")), + "reviewer_readiness": reviewer_readiness, + "source_owner_response_validation": { + "schema_version": validation.get("schema_version"), + "status": validation.get("status"), + "blocker_count": len(_list(validation.get("blockers"))), + "accepted_item_count": _dict(validation.get("result")).get( + "accepted_item_count" + ), + "projected_effective_escrow_missing_count": _dict( + validation.get("result") + ).get("projected_effective_escrow_missing_count"), + }, + "safe_next_step": ( + "independent_reviewer_acceptance_then_marker_dry_run" + if status == "accepted_for_independent_reviewer_readiness_only" + else "supplement_redacted_non_secret_evidence_refs_without_secret_values" + ), + } + + def _build_payload( owner_request: dict[str, Any], matrix: dict[str, Any], @@ -412,6 +516,13 @@ def _build_single_preflight_intake(generated_at: Any) -> dict[str, Any]: "owner_response_skeleton": _build_single_preflight_owner_response_skeleton( generated_at, ), + "evidence_refs_validation_endpoint": ( + "/api/v1/agents/credential-escrow-evidence-intake-readiness/" + "validate-evidence-refs" + ), + "evidence_refs_validation_mode": ( + "validate_redacted_non_secret_refs_only_no_persist_no_marker_write" + ), "single_preflight_command": ( "python3 scripts/reboot-recovery/post-reboot-owner-response-preflight.py " "--owner-packet-file " @@ -540,6 +651,115 @@ def _build_single_preflight_owner_response_skeleton( } +def _owner_response_from_evidence_refs(payload: dict[str, Any]) -> dict[str, Any]: + submissions = _normalized_evidence_ref_submissions(payload) + defaults = _dict(payload.get("defaults")) + owner_role = _field(payload, defaults, "owner_role", "owner_role_here") + owner_team = _field(payload, defaults, "owner_team", "owner_team_here") + reviewer = _field(payload, defaults, "reviewer", "reviewer_here") + last_reviewed_at = _field(payload, defaults, "last_reviewed_at", "pending") + followup_owner = _field(payload, defaults, "followup_owner", owner_role) + recovery_owner = _field(payload, defaults, "recovery_owner", owner_role) + evidence_refs = _strings(payload.get("redacted_evidence_refs")) + if not evidence_refs: + evidence_refs = [ + str(item.get("non_secret_evidence_ref") or "").strip() + for item in submissions + if not _is_placeholder(item.get("non_secret_evidence_ref")) + ] + item_by_id = { + str(item.get("item_id") or "").strip(): item + for item in submissions + if str(item.get("item_id") or "").strip() + } + return { + "schema_version": _OWNER_RESPONSE_SCHEMA, + "generated_at": payload.get("generated_at"), + "responses": [ + { + "gate_id": _GATE_ID, + "owner_role": owner_role, + "owner_team": owner_team, + "decision": str(payload.get("decision") or "accepted"), + "decision_reason": str( + payload.get("decision_reason") + or "validated redacted non-secret evidence refs only" + ), + "affected_scope": str( + payload.get("affected_scope") + or "P0-005 DR credential escrow evidence" + ), + "redacted_evidence_refs": evidence_refs, + "followup_owner": followup_owner, + "runtime_action_requested": False, + "runtime_action_authorized": False, + "host_write_requested": False, + "host_write_authorized": False, + "secret_value_included": False, + "secret_value_collection_allowed": False, + "credential_marker_write_requested": False, + "credential_marker_write_authorized": False, + "escrow_items": [ + _owner_response_escrow_item( + item_id, + _dict(item_by_id.get(item_id)), + recovery_owner, + reviewer, + last_reviewed_at, + ) + for item_id in _REQUIRED_ITEM_ORDER + ], + } + ], + } + + +def _owner_response_escrow_item( + item_id: str, + submission: dict[str, Any], + recovery_owner: str, + reviewer: str, + last_reviewed_at: str, +) -> dict[str, Any]: + return { + "item_id": item_id, + "non_secret_evidence_ref": str( + submission.get("non_secret_evidence_ref") or "non_secret_evidence_ref_here" + ), + "recovery_owner": str(submission.get("recovery_owner") or recovery_owner), + "reviewer": str(submission.get("reviewer") or reviewer), + "last_reviewed_at": str( + submission.get("last_reviewed_at") or last_reviewed_at + ), + "contains_secret_value": submission.get("contains_secret_value") is True, + } + + +def _normalized_evidence_ref_submissions(payload: dict[str, Any]) -> list[dict[str, Any]]: + raw = payload.get("evidence_refs") + if raw is None: + raw = payload.get("escrow_items") + if isinstance(raw, dict): + return [ + {"item_id": str(item_id), "non_secret_evidence_ref": str(ref)} + for item_id, ref in raw.items() + ] + return [ + _dict(item) + for item in _list(raw) + if isinstance(item, dict) + ] + + +def _field( + payload: dict[str, Any], + defaults: dict[str, Any], + key: str, + fallback: str, +) -> str: + return str(payload.get(key) or defaults.get(key) or fallback) + + def _normalize_item(item: Any) -> dict[str, Any]: data = _dict(item) return { diff --git a/apps/api/tests/test_credential_escrow_evidence_intake_readiness_api.py b/apps/api/tests/test_credential_escrow_evidence_intake_readiness_api.py index fa4c6742b..efe39dd5d 100644 --- a/apps/api/tests/test_credential_escrow_evidence_intake_readiness_api.py +++ b/apps/api/tests/test_credential_escrow_evidence_intake_readiness_api.py @@ -9,6 +9,7 @@ from fastapi.testclient import TestClient from src.api.v1.agents import router from src.services.credential_escrow_evidence_intake_readiness import ( load_latest_credential_escrow_evidence_intake_readiness, + validate_credential_escrow_evidence_refs, validate_credential_escrow_evidence_owner_response, ) @@ -80,6 +81,13 @@ def test_credential_escrow_evidence_intake_exposes_single_preflight_packet(): in intake["scorecard_command"] ) assert "projected_effective_escrow_missing_count=0" in intake["exit_criteria"] + assert intake["evidence_refs_validation_endpoint"] == ( + "/api/v1/agents/credential-escrow-evidence-intake-readiness/" + "validate-evidence-refs" + ) + assert intake["evidence_refs_validation_mode"] == ( + "validate_redacted_non_secret_refs_only_no_persist_no_marker_write" + ) owner_packet = intake["owner_packet"]["owner_packets"][0] assert owner_packet["packet_id"] == "credential_escrow_evidence" @@ -190,6 +198,76 @@ def test_credential_escrow_evidence_owner_response_validator_rejects_runtime_or_ assert payload["operation_boundaries"]["credential_marker_write_performed"] is False +def test_credential_escrow_evidence_refs_validator_accepts_simplified_refs(): + payload = validate_credential_escrow_evidence_refs( + _valid_evidence_refs_submission(), + readiness={ + "status": "blocked_waiting_non_secret_credential_escrow_evidence", + "rollups": {"effective_escrow_missing_count": 5}, + }, + ) + + assert payload["schema_version"] == "credential_escrow_evidence_refs_validation_v1" + assert payload["status"] == "accepted_for_independent_reviewer_readiness_only" + assert payload["scope"] == "credential_escrow_evidence_refs_intake" + assert payload["result"]["simplified_evidence_ref_submission_count"] == 5 + assert payload["result"]["accepted_item_count"] == 5 + assert payload["result"]["missing_required_item_ids"] == [] + assert payload["result"]["projected_effective_escrow_missing_count"] == 0 + assert payload["result"]["credential_marker_write_authorized_count"] == 0 + assert payload["result"]["payload_persisted"] is False + assert payload["operation_boundaries"]["payload_persisted"] is False + assert payload["operation_boundaries"]["credential_marker_write_performed"] is False + assert payload["operation_boundaries"]["runtime_action_performed"] is False + assert ( + payload["source_owner_response_validation"]["status"] + == "accepted_for_independent_reviewer_readiness_only" + ) + assert payload["safe_next_step"] == ( + "independent_reviewer_acceptance_then_marker_dry_run" + ) + + +def test_credential_escrow_evidence_refs_validator_quarantines_secret_ref(): + submission = _valid_evidence_refs_submission() + submission["evidence_refs"]["restic_repository_password"] = ( + "password=not-allowed" + ) + + payload = validate_credential_escrow_evidence_refs( + submission, + readiness={ + "status": "blocked_waiting_non_secret_credential_escrow_evidence", + "rollups": {"effective_escrow_missing_count": 5}, + }, + ) + + assert payload["status"] == "quarantined_sensitive_payload" + assert payload["result"]["owner_response_accepted_count"] == 0 + assert payload["result"]["source_sensitive_payload_hit_count"] == 1 + assert payload["operation_boundaries"]["payload_persisted"] is False + + +def test_credential_escrow_evidence_refs_endpoint_validates_without_persisting(): + app = FastAPI() + app.include_router(router, prefix="/api/v1") + client = TestClient(app) + + response = client.post( + "/api/v1/agents/credential-escrow-evidence-intake-readiness/" + "validate-evidence-refs", + json=_valid_evidence_refs_submission(), + ) + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "accepted_for_independent_reviewer_readiness_only" + assert data["result"]["projected_effective_escrow_missing_count"] == 0 + assert data["result"]["payload_persisted"] is False + assert data["operation_boundaries"]["secret_plaintext_read"] is False + assert data["operation_boundaries"]["credential_marker_write_performed"] is False + + def test_credential_escrow_evidence_owner_response_endpoint_validates_without_persisting(): app = FastAPI() app.include_router(router, prefix="/api/v1") @@ -331,3 +409,18 @@ def _valid_redacted_owner_response() -> dict: } ], } + + +def _valid_evidence_refs_submission() -> dict: + return { + "owner_role": "backup_dr_owner", + "owner_team": "platform_security", + "reviewer": "security_reviewer", + "last_reviewed_at": "2026-06-29", + "followup_owner": "backup_dr_owner", + "redacted_evidence_refs": ["review-ticket-20260629-p0-005"], + "evidence_refs": { + item_id: f"review-ticket-20260629-p0-005-{index}" + for index, item_id in enumerate(ESCROW_ITEMS) + }, + } diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index e894d814c..faea48a59 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,22 @@ +## 2026-06-29 — 20:36 P0-005 non-blocking evidence refs validator + +**照優先順序修正執行策略**： +- P0-006 仍是第一順位，但目前卡在 StockPlatform 官方 margin-short 2026-06-29 尚未發布 / retry window 等待；這種外部等待不應造成整條 P0 主線空轉。 +- 在不碰 secret、credential、`.env`、raw session、SQLite、GitHub、runtime marker、backup / restore / offsite sync、host/K8s write 的前提下，推進下一個可驗證 P0：P0-005 credential escrow evidence refs intake。 + +**完成內容**： +- 新增 `validate_credential_escrow_evidence_refs()`，支援只提供五個 `non_secret_evidence_ref` 與 reviewer metadata，就自動轉成既有 owner response preflight 結構。 +- 新增 POST `/api/v1/agents/credential-escrow-evidence-intake-readiness/validate-evidence-refs`。 +- `single_preflight_intake` 現在直接回傳此 validation endpoint 與 `validate_redacted_non_secret_refs_only_no_persist_no_marker_write` 模式，避免 P0-005 被誤讀成只能等完整 owner response JSON。 +- 更新 `docs/operations/awoooi-priority-work-order-readback.snapshot.json`：P0-006 等外部資料時，P0-005 可並行推進 no-persist evidence refs validation。 + +**驗證**： +- `pytest apps/api/tests/test_credential_escrow_evidence_intake_readiness_api.py apps/api/tests/test_delivery_closure_workbench_api.py scripts/reboot-recovery/tests/test_dr_escrow_evidence_checklist.py scripts/reboot-recovery/tests/test_post_reboot_credential_escrow_intake_scorecard.py`：25 passed。 +- `python -m py_compile apps/api/src/services/credential_escrow_evidence_intake_readiness.py apps/api/src/api/v1/agents.py`：通過。 +- `git diff --check`：通過。 + +**邊界**：此輪沒有保存 payload、沒有讀 credential、沒有寫 escrow marker、沒有 backup / restore / offsite sync、沒有 workflow trigger、沒有 host/K8s write、沒有讀 raw session / SQLite / auth / `.env`、沒有使用 GitHub / `gh` / GitHub API。 + ## 2026-06-29 — 20:30 P0-006 CD failure closeout **照優先順序處理**： diff --git a/docs/operations/awoooi-priority-work-order-readback.snapshot.json b/docs/operations/awoooi-priority-work-order-readback.snapshot.json index 535e2ce66..419e82439 100644 --- a/docs/operations/awoooi-priority-work-order-readback.snapshot.json +++ b/docs/operations/awoooi-priority-work-order-readback.snapshot.json @@ -398,6 +398,12 @@ "secret_value_collection_allowed": false, "source_safe_next_step": "collect_redacted_non_secret_evidence_refs_then_rerun_preflight", "source_top_level_safe_next_step_present": true, + "evidence_refs_validation_endpoint_present": true, + "evidence_refs_validation_endpoint": "/api/v1/agents/credential-escrow-evidence-intake-readiness/validate-evidence-refs", + "evidence_refs_validation_schema_version": "credential_escrow_evidence_refs_validation_v1", + "evidence_refs_validation_payload_persisted": false, + "evidence_refs_validation_credential_marker_write_performed": false, + "evidence_refs_validation_runtime_action_performed": false, "stock_blockers": [ "core_margin_short_daily_missing", "ai_recommendations_stale" @@ -444,8 +450,8 @@ ], "owner": "DR evidence lane" }, - "reason": "Backup/offsite core remains green and the credential escrow readiness API correctly exposes safe_next_step. Current product data freshness is still blocked after the 20:05 StockPlatform retry because official margin-short runs 3385/3386 remain official_pending; the separate DR blocker remains the five non-secret escrow evidence refs.", - "safe_next_step": "after_p0_006_stockplatform_freshness_clears_collect_five_non_secret_credential_escrow_evidence_refs_and_rerun_single_preflight", + "reason": "Backup/offsite core remains green and the credential escrow readiness API correctly exposes safe_next_step. Current product data freshness is still blocked after the 20:05 StockPlatform retry because official margin-short runs 3385/3386 remain official_pending; because P0-006 is waiting on an external official data window, P0-005 can proceed on its non-secret redacted refs validation path without opening runtime or marker writes.", + "safe_next_step": "while_p0_006_waits_external_stockplatform_retry_validate_five_non_secret_credential_escrow_evidence_refs_then_rerun_single_preflight", "status": "stock_freshness_blocked_and_waiting_non_secret_credential_escrow_evidence_refs", "title": "產品資料與備份 contract", "workplan_id": "P0-005" @@ -453,7 +459,8 @@ ], "next_execution_order": [ "P0-006: StockPlatform freshness/ingestion readback remains blocked after the 20:05 retry; wait the next retry windows, rerun verify-only with no reboot/restart/DB write, and open controlled data recovery gate only if still blocked after the final retry window.", - "P0-005: after P0-006 freshness clears, collect five non-secret credential escrow evidence refs and rerun one preflight; production API safe_next_step reads back correctly and backup/offsite core remains green.", + "P0-005: while P0-006 is waiting on external StockPlatform official data retry windows, validate five non-secret credential escrow evidence refs through the no-persist refs endpoint and rerun one preflight; production API safe_next_step reads back correctly and backup/offsite core remains green.", + "P0-003: supply authenticated/admin redacted inventory payload or owner coverage attestation, then run the Gitea payload validator; public-only live inventory remains four repos and GitHub remains stopped/retired/do_not_use.", "P1-OPENCLAW-LIVE-OPS-SPACE: build the Gather-like OpenClaw continuous animated live ops workspace only after the current P0 runtime truth blockers are no longer the active next action." ], "noise_integrated_risk_register": [