feat(api): validate harbor registry recovery receipts

apps/api/src/api/v1/agents.py

@@ -406,6 +406,9 @@ from src.services.github_target_private_backup_evidence_gate import (
 from src.services.harbor_registry_controlled_recovery_preflight import (
     load_latest_harbor_registry_controlled_recovery_preflight,
 )
+from src.services.harbor_registry_controlled_recovery_receipt import (
+    validate_harbor_registry_controlled_recovery_receipt,
+)
 from src.services.host_runaway_aiops_loop_readiness import (
     load_latest_host_runaway_aiops_loop_readiness,
 )
@@ -1213,6 +1216,38 @@ async def get_harbor_registry_controlled_recovery_preflight() -> dict[str, Any]:
         ) from exc
 
 
+@router.post(
+    "/harbor-registry-controlled-recovery-receipt",
+    response_model=dict[str, Any],
+    summary="驗證 Harbor registry controlled recovery receipt",
+    description=(
+        "驗證 110 local SSH repair / Harbor watchdog check / repair-once 的"
+        "非秘密輸出與 registry /v2/ verifier，回傳可供 KM/RAG/MCP/PlayBook "
+        "metadata writeback 的 receipt。此端點不回傳 raw log、不 SSH、不 Docker、"
+        "不 restart、不讀 secret、不觸發 workflow、不寫 runtime。"
+    ),
+)
+async def validate_harbor_registry_controlled_recovery_receipt_packet(
+    receipt_payload: dict[str, Any],
+) -> dict[str, Any]:
+    """驗證 Harbor registry controlled recovery receipt。"""
+    try:
+        payload = await asyncio.to_thread(
+            validate_harbor_registry_controlled_recovery_receipt,
+            receipt_payload,
+        )
+        return redact_public_lan_topology(payload)
+    except ValueError as exc:
+        logger.error(
+            "harbor_registry_controlled_recovery_receipt_invalid",
+            error=str(exc),
+        )
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(exc),
+        ) from exc
+
+
 @router.get(
     "/product-awoooi-manifest-standard",
     response_model=dict[str, Any],

apps/api/src/services/harbor_registry_controlled_recovery_preflight.py

@@ -163,6 +163,9 @@ def _build_payload(
             ),
             "watchdog_check_command": "harbor-watchdog.sh --check",
             "watchdog_repair_command": "harbor-watchdog.sh --repair-once",
+            "receipt_validator_endpoint": (
+                "/api/v1/agents/harbor-registry-controlled-recovery-receipt"
+            ),
         },
         "source_of_truth_diff": {
             "status": _source_diff_status(
@@ -487,6 +490,13 @@ def _controlled_work_items(
             "status": "ready",
             "required_output": "metadata_only_registry_recovery_receipt_and_trust_delta",
         },
+        {
+            "id": "harbor-registry-recovery-receipt-validator",
+            "title": "Controlled recovery receipt validator",
+            "owner_agent": "Verifier Agent",
+            "status": "ready",
+            "required_output": "harbor_registry_controlled_recovery_receipt_v1",
+        },
     ]
 
 
@@ -562,6 +572,11 @@ def _post_apply_verifiers() -> list[dict[str, Any]]:
             "url": "/api/v1/agents/awoooi-priority-work-order-readback",
             "expected_rollup": "production deploy marker advances after Harbor recovery",
         },
+        {
+            "id": "harbor_registry_controlled_recovery_receipt",
+            "url": "/api/v1/agents/harbor-registry-controlled-recovery-receipt",
+            "expected_status": "harbor_registry_recovery_receipt_verified",
+        },
     ]
 
 

apps/api/src/services/harbor_registry_controlled_recovery_receipt.py

@@ -0,0 +1,346 @@
+"""Harbor registry controlled recovery receipt validation.
+
+This validator turns non-secret 110 local repair/watchdog output plus public
+registry verifiers into a machine-readable receipt. It never executes SSH,
+Docker, systemctl, workflow dispatch, or runtime writes, and it does not echo
+raw log text back to callers.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+_SCHEMA_VERSION = "harbor_registry_controlled_recovery_receipt_v1"
+_READY_HTTP_STATUSES = {200, 401}
+
+
+def validate_harbor_registry_controlled_recovery_receipt(
+    receipt_payload: dict[str, Any],
+) -> dict[str, Any]:
+    """Validate a Harbor registry controlled recovery receipt payload."""
+    if not isinstance(receipt_payload, dict):
+        raise ValueError("receipt payload must be a JSON object")
+
+    ssh_local_output = _text(receipt_payload.get("ssh_local_repair_output"))
+    watchdog_check_output = _text(receipt_payload.get("watchdog_check_output"))
+    watchdog_repair_output = _text(receipt_payload.get("watchdog_repair_output"))
+
+    ssh_local = _parse_ssh_local_repair_output(ssh_local_output)
+    watchdog_check = _parse_watchdog_output(watchdog_check_output)
+    watchdog_repair = _parse_watchdog_output(watchdog_repair_output)
+    verifier = _post_apply_verifier(receipt_payload)
+
+    active_blockers = _active_blockers(
+        ssh_local=ssh_local,
+        watchdog_check=watchdog_check,
+        watchdog_repair=watchdog_repair,
+        verifier=verifier,
+    )
+    status = _status(
+        ssh_local=ssh_local,
+        watchdog_check=watchdog_check,
+        watchdog_repair=watchdog_repair,
+        verifier=verifier,
+        active_blockers=active_blockers,
+    )
+
+    return {
+        "schema_version": _SCHEMA_VERSION,
+        "priority": "P0-006",
+        "scope": "harbor_registry_controlled_recovery_receipt",
+        "status": status,
+        "accepted": status == "harbor_registry_recovery_receipt_verified",
+        "safe_next_step": _safe_next_step(status=status),
+        "active_blockers": active_blockers,
+        "active_blocker_count": len(active_blockers),
+        "input_redaction": {
+            "raw_output_returned": False,
+            "ssh_local_repair_output": _text_stats(ssh_local_output),
+            "watchdog_check_output": _text_stats(watchdog_check_output),
+            "watchdog_repair_output": _text_stats(watchdog_repair_output),
+        },
+        "readback": {
+            "ssh_local_repair": ssh_local,
+            "watchdog_check": watchdog_check,
+            "watchdog_repair": watchdog_repair,
+            "post_apply_verifier": verifier,
+        },
+        "controlled_apply_policy": {
+            "risk_level": "high",
+            "break_glass_required": False,
+            "manual_end_state": False,
+            "current_apply_allowed": False,
+            "current_apply_blocker": _current_apply_blocker(status=status),
+            "allowed_scope": [
+                "validate_non_secret_ssh_local_repair_receipt",
+                "validate_harbor_watchdog_check_receipt",
+                "validate_harbor_watchdog_repair_once_receipt",
+                "validate_public_and_internal_registry_v2_verifier",
+                "km_rag_mcp_playbook_metadata_writeback",
+                "retry_gitea_cd_after_registry_v2_green",
+            ],
+            "forbidden_scope": [
+                "secret_read",
+                "docker_daemon_restart",
+                "host_reboot",
+                "node_drain",
+                "nginx_reload_or_restart",
+                "database_restore_or_prune",
+                "workflow_dispatch_from_receipt_validator",
+                "force_push_or_repo_ref_mutation",
+                "registry_provider_switch",
+            ],
+        },
+        "learning_writeback_contracts": _learning_writeback_contracts(status=status),
+        "rollups": {
+            "ssh_local_repair_receipt_seen": ssh_local["receipt_seen"],
+            "ssh_local_repair_control_channel_metadata_ready": ssh_local[
+                "control_channel_metadata_ready"
+            ],
+            "watchdog_check_receipt_seen": watchdog_check["receipt_seen"],
+            "watchdog_check_harbor_ready": watchdog_check["harbor_ready"],
+            "watchdog_repair_receipt_seen": watchdog_repair["receipt_seen"],
+            "watchdog_repair_harbor_ready": watchdog_repair["harbor_ready"],
+            "post_apply_verifier_ready": verifier["registry_v2_ready"],
+            "metadata_writeback_contract_ready": True,
+        },
+        "operation_boundaries": {
+            "raw_output_returned": False,
+            "ssh_used": False,
+            "docker_command_performed": False,
+            "docker_restart_performed": False,
+            "docker_daemon_restart_performed": False,
+            "host_reboot_performed": False,
+            "node_drain_performed": False,
+            "nginx_reload_or_restart_performed": False,
+            "database_write_or_restore_performed": False,
+            "secret_value_collection_allowed": False,
+            "github_api_used": False,
+            "workflow_trigger_performed": False,
+            "runtime_write_allowed_by_this_validator": False,
+        },
+    }
+
+
+def _parse_ssh_local_repair_output(output: str) -> dict[str, Any]:
+    fields = _parse_key_values(output)
+    marker_seen = "AWOOOI_110_SSH_PUBLICKEY_AUTH_LOCAL_REPAIR" in output
+    sshd_ok = "SSHD_CONFIG_SYNTAX=ok" in output
+    sshd_after_ok = (
+        "SSHD_CONFIG_SYNTAX_AFTER_APPLY=ok" in output
+        or not _bool_from_field(fields.get("APPLY"))
+    )
+    user_exists = "USER_STATUS user=" in output and "exists=1" in output
+    authorized_keys_exists = (
+        "AUTHORIZED_KEYS_STATUS" in output and "exists=1" in output
+    )
+    permissions_applied = "APPLIED permissions" in output
+    reload_done = "SSH_RELOAD=done" in output
+    reload_skipped = "SSH_RELOAD=skipped" in output
+    return {
+        "receipt_seen": marker_seen,
+        "mode": _mode_from_marker_line(output),
+        "sshd_config_syntax_ok": sshd_ok,
+        "sshd_config_syntax_after_apply_ok": sshd_after_ok,
+        "target_user_exists": user_exists,
+        "authorized_keys_metadata_present": authorized_keys_exists,
+        "permissions_applied": permissions_applied,
+        "ssh_reload_done": reload_done,
+        "ssh_reload_skipped": reload_skipped,
+        "control_channel_metadata_ready": bool(
+            marker_seen and sshd_ok and sshd_after_ok and user_exists and authorized_keys_exists
+        ),
+    }
+
+
+def _parse_watchdog_output(output: str) -> dict[str, Any]:
+    fields = _parse_key_values(output)
+    marker_seen = "AWOOOI_HARBOR_WATCHDOG_CHECK" in output
+    harbor_ready = _bool_from_field(fields.get("harbor_ready"))
+    local_status = _int_or_none(fields.get("harbor_local_v2_http_status"))
+    docker_action = _bool_from_field(fields.get("docker_compose_action_performed"))
+    service_restart = _bool_from_field(fields.get("service_restart_performed"))
+    host_reboot = _bool_from_field(fields.get("host_reboot_performed"))
+    expected_host = _bool_from_field(fields.get("expected_host_ip_present"))
+    repair_attempt_seen = (
+        "Harbor Watchdog" in output
+        or "Harbor 修復" in output
+        or "docker compose up -d" in output
+    )
+    return {
+        "receipt_seen": marker_seen,
+        "mode": str(fields.get("mode") or ""),
+        "expected_host_ip_present": expected_host,
+        "harbor_local_v2_http_status": local_status,
+        "harbor_ready": harbor_ready,
+        "check_only": _bool_from_field(fields.get("check_only")),
+        "repair_attempt_seen": repair_attempt_seen,
+        "docker_compose_action_performed": docker_action,
+        "service_restart_performed": service_restart,
+        "host_reboot_performed": host_reboot,
+        "forbidden_action_seen": bool(service_restart or host_reboot),
+    }
+
+
+def _post_apply_verifier(receipt_payload: dict[str, Any]) -> dict[str, Any]:
+    public_status = _int_or_none(receipt_payload.get("public_registry_v2_http_status"))
+    internal_status = _int_or_none(
+        receipt_payload.get("internal_registry_v2_http_status")
+    )
+    public_ready = public_status in _READY_HTTP_STATUSES
+    internal_ready = internal_status in _READY_HTTP_STATUSES
+    return {
+        "public_registry_v2_http_status": public_status,
+        "internal_registry_v2_http_status": internal_status,
+        "public_registry_v2_ready": public_ready,
+        "internal_registry_v2_ready": internal_ready,
+        "registry_v2_ready": public_ready and internal_ready,
+    }
+
+
+def _active_blockers(
+    *,
+    ssh_local: dict[str, Any],
+    watchdog_check: dict[str, Any],
+    watchdog_repair: dict[str, Any],
+    verifier: dict[str, Any],
+) -> list[str]:
+    blockers: list[str] = []
+    if ssh_local["receipt_seen"] and not ssh_local["control_channel_metadata_ready"]:
+        blockers.append("ssh_local_repair_receipt_metadata_not_ready")
+    if not watchdog_check["receipt_seen"]:
+        blockers.append("harbor_watchdog_check_receipt_missing")
+    elif watchdog_check["forbidden_action_seen"]:
+        blockers.append("harbor_watchdog_check_forbidden_action_seen")
+    elif not watchdog_check["harbor_ready"] and not watchdog_repair["receipt_seen"]:
+        blockers.append("harbor_watchdog_repair_once_receipt_missing")
+    if watchdog_repair["receipt_seen"] and watchdog_repair["forbidden_action_seen"]:
+        blockers.append("harbor_watchdog_repair_forbidden_action_seen")
+    if watchdog_repair["receipt_seen"] and not watchdog_repair["harbor_ready"]:
+        blockers.append("harbor_watchdog_repair_did_not_restore_local_v2")
+    if not verifier["public_registry_v2_ready"]:
+        blockers.append("public_registry_v2_verifier_not_green")
+    if not verifier["internal_registry_v2_ready"]:
+        blockers.append("internal_registry_v2_verifier_not_green")
+    return _unique_strings(blockers)
+
+
+def _status(
+    *,
+    ssh_local: dict[str, Any],
+    watchdog_check: dict[str, Any],
+    watchdog_repair: dict[str, Any],
+    verifier: dict[str, Any],
+    active_blockers: list[str],
+) -> str:
+    if verifier["registry_v2_ready"] and not active_blockers:
+        return "harbor_registry_recovery_receipt_verified"
+    if watchdog_repair["receipt_seen"]:
+        return "harbor_registry_repair_receipt_waiting_registry_v2_verifier"
+    if watchdog_check["receipt_seen"] and watchdog_check["harbor_ready"]:
+        return "harbor_local_registry_ready_waiting_public_registry_v2_verifier"
+    if watchdog_check["receipt_seen"]:
+        return "harbor_watchdog_check_unhealthy_waiting_repair_once_receipt"
+    if ssh_local["receipt_seen"]:
+        return "ssh_local_repair_receipt_waiting_harbor_watchdog_check"
+    return "blocked_waiting_harbor_controlled_recovery_receipt"
+
+
+def _safe_next_step(*, status: str) -> str:
+    if status == "harbor_registry_recovery_receipt_verified":
+        return "retry_gitea_cd_then_verify_deploy_marker_and_priority_readback"
+    if status == "harbor_registry_repair_receipt_waiting_registry_v2_verifier":
+        return "rerun_public_and_internal_registry_v2_verifier_before_cd_retry"
+    if status == "harbor_local_registry_ready_waiting_public_registry_v2_verifier":
+        return "verify_public_registry_v2_route_then_retry_gitea_cd"
+    if status == "harbor_watchdog_check_unhealthy_waiting_repair_once_receipt":
+        return "run_harbor_watchdog_repair_once_then_submit_receipt_with_verifier"
+    if status == "ssh_local_repair_receipt_waiting_harbor_watchdog_check":
+        return "rerun_ssh_control_channel_probe_then_harbor_watchdog_check_mode"
+    return "submit_non_secret_ssh_local_or_harbor_watchdog_receipt"
+
+
+def _current_apply_blocker(*, status: str) -> str:
+    if status == "harbor_registry_recovery_receipt_verified":
+        return "receipt_verified_waiting_cd_marker_readback"
+    if status == "harbor_watchdog_check_unhealthy_waiting_repair_once_receipt":
+        return "repair_once_receipt_required_after_unhealthy_check"
+    if status == "ssh_local_repair_receipt_waiting_harbor_watchdog_check":
+        return "harbor_watchdog_check_receipt_required_after_ssh_local_repair"
+    return "registry_v2_verifier_or_controlled_receipt_required"
+
+
+def _learning_writeback_contracts(*, status: str) -> list[dict[str, Any]]:
+    targets = [
+        "timeline_event",
+        "knowledge_entry",
+        "rag_embedding",
+        "mcp_tool_registry_signal",
+        "playbook_trust_score",
+        "controlled_recovery_report",
+    ]
+    return [
+        {
+            "target": target,
+            "writeback_mode": "metadata_only",
+            "raw_log_allowed": False,
+            "secret_value_allowed": False,
+            "receipt_key": f"harbor_registry_recovery_receipt::{status}::{target}",
+        }
+        for target in targets
+    ]
+
+
+def _parse_key_values(output: str) -> dict[str, str]:
+    fields: dict[str, str] = {}
+    for raw_line in output.splitlines():
+        for token in raw_line.strip().split():
+            if "=" not in token:
+                continue
+            key, value = token.split("=", 1)
+            if key:
+                fields[key] = value
+    return fields
+
+
+def _mode_from_marker_line(output: str) -> str:
+    for raw_line in output.splitlines():
+        if "AWOOOI_110_SSH_PUBLICKEY_AUTH_LOCAL_REPAIR" not in raw_line:
+            continue
+        fields = _parse_key_values(raw_line)
+        return str(fields.get("mode") or "")
+    return ""
+
+
+def _text(value: Any) -> str:
+    return value if isinstance(value, str) else ""
+
+
+def _text_stats(value: str) -> dict[str, Any]:
+    return {
+        "provided": bool(value),
+        "byte_count": len(value.encode("utf-8")),
+        "line_count": len(value.splitlines()) if value else 0,
+    }
+
+
+def _bool_from_field(value: Any) -> bool:
+    return str(value).strip().lower() in {"1", "true", "yes", "ok", "done"}
+
+
+def _int_or_none(value: Any) -> int | None:
+    try:
+        return int(str(value))
+    except (TypeError, ValueError):
+        return None
+
+
+def _unique_strings(values: list[str]) -> list[str]:
+    seen: set[str] = set()
+    unique: list[str] = []
+    for value in values:
+        if value in seen:
+            continue
+        seen.add(value)
+        unique.append(value)
+    return unique

apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from src.api.v1.agents import router
+from src.services.harbor_registry_controlled_recovery_receipt import (
+    validate_harbor_registry_controlled_recovery_receipt,
+)
+
+
+def test_harbor_recovery_receipt_accepts_verified_repair() -> None:
+    payload = validate_harbor_registry_controlled_recovery_receipt(
+        {
+            "ssh_local_repair_output": _ssh_local_apply_output(),
+            "watchdog_check_output": _watchdog_check_output(ready=False, status=502),
+            "watchdog_repair_output": _watchdog_check_output(ready=True, status=401),
+            "public_registry_v2_http_status": 401,
+            "internal_registry_v2_http_status": 401,
+        }
+    )
+
+    assert payload["schema_version"] == (
+        "harbor_registry_controlled_recovery_receipt_v1"
+    )
+    assert payload["status"] == "harbor_registry_recovery_receipt_verified"
+    assert payload["accepted"] is True
+    assert payload["active_blockers"] == []
+    assert payload["readback"]["ssh_local_repair"][
+        "control_channel_metadata_ready"
+    ] is True
+    assert payload["readback"]["watchdog_repair"]["harbor_ready"] is True
+    assert payload["readback"]["post_apply_verifier"]["registry_v2_ready"] is True
+    assert payload["input_redaction"]["raw_output_returned"] is False
+    assert payload["operation_boundaries"]["ssh_used"] is False
+    assert payload["operation_boundaries"]["docker_command_performed"] is False
+    assert payload["operation_boundaries"]["host_reboot_performed"] is False
+    assert payload["operation_boundaries"]["workflow_trigger_performed"] is False
+    assert payload["learning_writeback_contracts"][0]["raw_log_allowed"] is False
+
+
+def test_harbor_recovery_receipt_routes_unhealthy_check_to_repair_once() -> None:
+    payload = validate_harbor_registry_controlled_recovery_receipt(
+        {
+            "watchdog_check_output": _watchdog_check_output(
+                ready=False,
+                status=502,
+            ),
+            "public_registry_v2_http_status": 502,
+            "internal_registry_v2_http_status": 502,
+        }
+    )
+
+    assert payload["status"] == (
+        "harbor_watchdog_check_unhealthy_waiting_repair_once_receipt"
+    )
+    assert "harbor_watchdog_repair_once_receipt_missing" in payload[
+        "active_blockers"
+    ]
+    assert payload["safe_next_step"] == (
+        "run_harbor_watchdog_repair_once_then_submit_receipt_with_verifier"
+    )
+    assert payload["controlled_apply_policy"]["manual_end_state"] is False
+
+
+def test_harbor_recovery_receipt_endpoint_redacts_raw_output() -> None:
+    app = FastAPI()
+    app.include_router(router, prefix="/api/v1")
+    client = TestClient(app)
+
+    response = client.post(
+        "/api/v1/agents/harbor-registry-controlled-recovery-receipt",
+        json={
+            "watchdog_check_output": _watchdog_check_output(
+                ready=True,
+                status=401,
+            ),
+            "public_registry_v2_http_status": 401,
+            "internal_registry_v2_http_status": 401,
+        },
+    )
+
+    assert response.status_code == 200
+    data = response.json()
+    assert data["status"] == "harbor_registry_recovery_receipt_verified"
+    assert data["accepted"] is True
+    assert "watchdog_check_output" not in str(data["readback"])
+    assert data["input_redaction"]["watchdog_check_output"]["line_count"] > 0
+
+
+def _ssh_local_apply_output() -> str:
+    return """
+AWOOOI_110_SSH_PUBLICKEY_AUTH_LOCAL_REPAIR mode=apply target_user=wooo
+SSH_SERVICE_ACTIVE=active
+SSHD_CONFIG_SYNTAX=ok
+USER_STATUS user=wooo exists=1 home=/home/wooo
+PATH_STATUS path=/home/wooo mode=755 owner=wooo group=wooo type=directory
+PATH_STATUS path=/home/wooo/.ssh mode=700 owner=wooo group=wooo type=directory
+AUTHORIZED_KEYS_STATUS path=/home/wooo/.ssh/authorized_keys exists=1 bytes=380 lines=1
+APPLIED permissions target_user=wooo home=/home/wooo
+SSHD_CONFIG_SYNTAX_AFTER_APPLY=ok
+SSH_RELOAD=skipped
+"""
+
+
+def _watchdog_check_output(*, ready: bool, status: int) -> str:
+    ready_text = "true" if ready else "false"
+    return f"""
+AWOOOI_HARBOR_WATCHDOG_CHECK
+mode=check
+target=127.0.0.1:5000/v2/
+expected_host_ip=192.168.0.110
+expected_host_ip_present=true
+harbor_dir=/home/wooo/harbor/harbor
+harbor_dir_exists=true
+harbor_compose_exists=true
+docker_cli_available=true
+docker_service_state=active
+lockfile=/var/lock/harbor-repair.lock
+lockfile_exists=false
+harbor_local_v2_http_status={status}
+harbor_ready={ready_text}
+check_only=true
+docker_compose_action_performed=false
+container_remove_performed=false
+service_restart_performed=false
+host_reboot_performed=false
+"""