diff --git a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py index 8b7c3c0ed..0ad69b38d 100644 --- a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py @@ -29,18 +29,23 @@ def validate_harbor_registry_controlled_recovery_receipt( watchdog_check = _parse_watchdog_output(watchdog_check_output) watchdog_repair = _parse_watchdog_output(watchdog_repair_output) verifier = _post_apply_verifier(receipt_payload) + gitea_queue = _gitea_queue_readback( + receipt_payload.get("gitea_actions_queue_readback") + ) active_blockers = _active_blockers( ssh_local=ssh_local, watchdog_check=watchdog_check, watchdog_repair=watchdog_repair, verifier=verifier, + gitea_queue=gitea_queue, ) status = _status( ssh_local=ssh_local, watchdog_check=watchdog_check, watchdog_repair=watchdog_repair, verifier=verifier, + gitea_queue=gitea_queue, active_blockers=active_blockers, ) local_console_phase_readback = _local_console_phase_readback( @@ -64,12 +69,19 @@ def validate_harbor_registry_controlled_recovery_receipt( "ssh_local_repair_output": _text_stats(ssh_local_output), "watchdog_check_output": _text_stats(watchdog_check_output), "watchdog_repair_output": _text_stats(watchdog_repair_output), + "gitea_actions_queue_readback": { + "provided": gitea_queue["receipt_seen"], + "metadata_only": True, + "raw_output_returned": False, + "top_visible_runs_returned": False, + }, }, "readback": { "ssh_local_repair": ssh_local, "watchdog_check": watchdog_check, "watchdog_repair": watchdog_repair, "post_apply_verifier": verifier, + "gitea_actions_queue": gitea_queue, }, "local_console_phase_readback": local_console_phase_readback, "controlled_apply_policy": { @@ -109,6 +121,20 @@ def validate_harbor_registry_controlled_recovery_receipt( "watchdog_repair_receipt_seen": watchdog_repair["receipt_seen"], "watchdog_repair_harbor_ready": watchdog_repair["harbor_ready"], "post_apply_verifier_ready": verifier["registry_v2_ready"], + "gitea_queue_readback_seen": gitea_queue["receipt_seen"], + "gitea_queue_blocker_count": gitea_queue["blocker_count"], + "gitea_queue_harbor_110_no_matching_runner": gitea_queue[ + "harbor_110_repair_no_matching_runner" + ], + "gitea_queue_harbor_110_jobs_stale_or_mismatched": gitea_queue[ + "harbor_110_repair_jobs_stale_or_mismatched" + ], + "gitea_queue_latest_cd_run_status": gitea_queue[ + "latest_visible_cd_run_status" + ], + "gitea_queue_harbor_110_repair_run_status": gitea_queue[ + "latest_visible_harbor_110_repair_run_status" + ], "metadata_writeback_contract_ready": True, "local_console_phase_count": local_console_phase_readback["phase_count"], "local_console_completed_phase_count": local_console_phase_readback[ @@ -317,12 +343,136 @@ def _post_apply_verifier(receipt_payload: dict[str, Any]) -> dict[str, Any]: } +def _gitea_queue_readback(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + return { + "receipt_seen": False, + "schema_version": "", + "status": "not_provided", + "latest_visible_cd_run_id": "", + "latest_visible_cd_run_status": "", + "latest_visible_cd_run_commit_sha": "", + "latest_visible_harbor_110_repair_run_id": "", + "latest_visible_harbor_110_repair_run_status": "", + "harbor_110_repair_no_matching_runner_label": "", + "harbor_110_repair_no_matching_runner": False, + "harbor_110_repair_waiting": False, + "harbor_110_repair_blocked": False, + "harbor_110_repair_jobs_stale_or_mismatched": False, + "harbor_110_repair_jobs_unexpected_names": [], + "harbor_110_repair_jobs_labels": [], + "harbor_110_repair_jobs_runner_names": [], + "operation_boundary_violation": False, + "blockers": [], + "blocker_count": 0, + "metadata_only": True, + "raw_output_returned": False, + "top_visible_runs_returned": False, + } + + readback = _dict(value.get("readback")) + rollups = _dict(value.get("rollups")) + operation_boundaries = _dict(value.get("operation_boundaries")) + no_matching_label = str( + rollups.get("harbor_110_repair_no_matching_runner_label") + or readback.get("latest_visible_harbor_110_repair_no_matching_runner_label") + or "" + ) + jobs_stale = bool( + rollups.get("harbor_110_repair_jobs_stale_or_mismatched") is True + or readback.get("harbor_110_repair_jobs_stale_or_mismatched") is True + ) + waiting = bool( + rollups.get("harbor_110_repair_waiting") is True + or readback.get("latest_visible_harbor_110_repair_waiting") is True + ) + blocked = bool( + rollups.get("harbor_110_repair_blocked") is True + or readback.get("latest_visible_harbor_110_repair_blocked") is True + ) + boundary_violation = any( + operation_boundaries.get(flag) is True + for flag in ( + "github_api_used", + "host_write_performed", + "runner_registration_performed", + "runner_service_start_performed", + "secret_or_runner_token_read", + "workflow_dispatch_performed", + ) + ) + blockers = _gitea_queue_blockers( + no_matching_runner=bool(no_matching_label), + jobs_stale=jobs_stale, + blocked=blocked, + boundary_violation=boundary_violation, + ) + return { + "receipt_seen": True, + "schema_version": str(value.get("schema_version") or ""), + "status": str(value.get("status") or "unknown"), + "latest_visible_cd_run_id": str(readback.get("latest_visible_cd_run_id") or ""), + "latest_visible_cd_run_status": str( + readback.get("latest_visible_cd_run_status") or "" + ), + "latest_visible_cd_run_commit_sha": str( + readback.get("latest_visible_cd_run_commit_sha") or "" + ), + "latest_visible_harbor_110_repair_run_id": str( + readback.get("latest_visible_harbor_110_repair_run_id") or "" + ), + "latest_visible_harbor_110_repair_run_status": str( + readback.get("latest_visible_harbor_110_repair_run_status") or "" + ), + "harbor_110_repair_no_matching_runner_label": no_matching_label, + "harbor_110_repair_no_matching_runner": bool(no_matching_label), + "harbor_110_repair_waiting": waiting, + "harbor_110_repair_blocked": blocked, + "harbor_110_repair_jobs_stale_or_mismatched": jobs_stale, + "harbor_110_repair_jobs_unexpected_names": _strings( + readback.get("harbor_110_repair_jobs_unexpected_names") + ), + "harbor_110_repair_jobs_labels": _strings( + readback.get("harbor_110_repair_jobs_labels") + ), + "harbor_110_repair_jobs_runner_names": _strings( + readback.get("harbor_110_repair_jobs_runner_names") + ), + "operation_boundary_violation": boundary_violation, + "blockers": blockers, + "blocker_count": len(blockers), + "metadata_only": True, + "raw_output_returned": False, + "top_visible_runs_returned": False, + } + + +def _gitea_queue_blockers( + *, + no_matching_runner: bool, + jobs_stale: bool, + blocked: bool, + boundary_violation: bool, +) -> list[str]: + blockers: list[str] = [] + if no_matching_runner: + blockers.append("gitea_queue_harbor_110_repair_no_matching_runner") + elif blocked: + blockers.append("gitea_queue_harbor_110_repair_blocked") + if jobs_stale: + blockers.append("gitea_queue_harbor_110_repair_jobs_stale_or_mismatched") + if boundary_violation: + blockers.append("gitea_queue_readback_boundary_violation") + return blockers + + def _active_blockers( *, ssh_local: dict[str, Any], watchdog_check: dict[str, Any], watchdog_repair: dict[str, Any], verifier: dict[str, Any], + gitea_queue: dict[str, Any], ) -> list[str]: blockers: list[str] = [] if ssh_local["receipt_seen"] and not ssh_local["control_channel_metadata_ready"]: @@ -341,6 +491,7 @@ def _active_blockers( blockers.append("public_registry_v2_verifier_not_green") if not verifier["internal_registry_v2_ready"]: blockers.append("internal_registry_v2_verifier_not_green") + blockers.extend(_strings(gitea_queue.get("blockers"))) return _unique_strings(blockers) @@ -350,10 +501,17 @@ def _status( watchdog_check: dict[str, Any], watchdog_repair: dict[str, Any], verifier: dict[str, Any], + gitea_queue: dict[str, Any], active_blockers: list[str], ) -> str: if verifier["registry_v2_ready"] and not active_blockers: return "harbor_registry_recovery_receipt_verified" + if ( + verifier["registry_v2_ready"] + and gitea_queue["receipt_seen"] + and gitea_queue["blocker_count"] > 0 + ): + return "harbor_registry_recovery_receipt_verified_waiting_gitea_queue_clearance" if watchdog_repair["receipt_seen"]: return "harbor_registry_repair_receipt_waiting_registry_v2_verifier" if watchdog_check["receipt_seen"] and watchdog_check["harbor_ready"]: @@ -368,6 +526,8 @@ def _status( def _safe_next_step(*, status: str) -> str: if status == "harbor_registry_recovery_receipt_verified": return "retry_gitea_cd_then_verify_deploy_marker_and_priority_readback" + if status == "harbor_registry_recovery_receipt_verified_waiting_gitea_queue_clearance": + return "clear_harbor_110_runner_queue_then_retry_cd_marker_readback" if status == "harbor_registry_repair_receipt_waiting_registry_v2_verifier": return "rerun_public_and_internal_registry_v2_verifier_before_cd_retry" if status == "harbor_local_registry_ready_waiting_public_registry_v2_verifier": @@ -382,6 +542,8 @@ def _safe_next_step(*, status: str) -> str: def _current_apply_blocker(*, status: str) -> str: if status == "harbor_registry_recovery_receipt_verified": return "receipt_verified_waiting_cd_marker_readback" + if status == "harbor_registry_recovery_receipt_verified_waiting_gitea_queue_clearance": + return "gitea_queue_clearance_required_after_registry_receipt" if status == "harbor_watchdog_check_unhealthy_waiting_repair_once_receipt": return "repair_once_receipt_required_after_unhealthy_check" if status == "ssh_local_repair_receipt_waiting_harbor_watchdog_check": @@ -454,6 +616,16 @@ def _int_or_none(value: Any) -> int | None: return None +def _dict(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _strings(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if str(item)] + + def _unique_strings(values: list[str]) -> list[str]: seen: set[str] = set() unique: list[str] = [] diff --git a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py index 974276306..842078d76 100644 --- a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py @@ -93,6 +93,51 @@ def test_harbor_recovery_receipt_routes_unhealthy_check_to_repair_once() -> None assert payload["controlled_apply_policy"]["manual_end_state"] is False +def test_harbor_recovery_receipt_surfaces_gitea_queue_blockers() -> None: + payload = validate_harbor_registry_controlled_recovery_receipt( + { + "watchdog_check_output": _watchdog_check_output( + ready=True, + status=401, + ), + "public_registry_v2_http_status": 401, + "internal_registry_v2_http_status": 401, + "gitea_actions_queue_readback": _gitea_queue_no_matching_runner(), + } + ) + + assert payload["status"] == ( + "harbor_registry_recovery_receipt_verified_waiting_gitea_queue_clearance" + ) + assert payload["accepted"] is False + assert payload["safe_next_step"] == ( + "clear_harbor_110_runner_queue_then_retry_cd_marker_readback" + ) + assert payload["active_blockers"] == [ + "gitea_queue_harbor_110_repair_no_matching_runner", + "gitea_queue_harbor_110_repair_jobs_stale_or_mismatched", + ] + queue = payload["readback"]["gitea_actions_queue"] + assert queue["status"] == "blocked_harbor_110_repair_no_matching_runner" + assert queue["harbor_110_repair_no_matching_runner_label"] == "awoooi-host" + assert queue["harbor_110_repair_jobs_stale_or_mismatched"] is True + assert queue["harbor_110_repair_jobs_unexpected_names"] == ["ai-code-review"] + assert queue["raw_output_returned"] is False + assert queue["top_visible_runs_returned"] is False + assert payload["rollups"]["gitea_queue_blocker_count"] == 2 + assert payload["rollups"]["gitea_queue_harbor_110_no_matching_runner"] is True + assert ( + payload["rollups"]["gitea_queue_harbor_110_jobs_stale_or_mismatched"] + is True + ) + assert payload["input_redaction"]["gitea_actions_queue_readback"] == { + "provided": True, + "metadata_only": True, + "raw_output_returned": False, + "top_visible_runs_returned": False, + } + + def test_harbor_recovery_receipt_endpoint_redacts_raw_output() -> None: app = FastAPI() app.include_router(router, prefix="/api/v1") @@ -156,3 +201,43 @@ container_remove_performed=false service_restart_performed=false host_reboot_performed=false """ + + +def _gitea_queue_no_matching_runner() -> dict: + return { + "schema_version": "awoooi_public_gitea_actions_queue_readback_v1", + "status": "blocked_harbor_110_repair_no_matching_runner", + "operation_boundaries": { + "public_gitea_read_only": True, + "github_api_used": False, + "host_write_performed": False, + "runner_registration_performed": False, + "runner_service_start_performed": False, + "secret_or_runner_token_read": False, + "workflow_dispatch_performed": False, + }, + "readback": { + "latest_visible_cd_run_id": "4110", + "latest_visible_cd_run_status": "Running", + "latest_visible_cd_run_commit_sha": ( + "5d10a9d1105757149b03373e35fc85a5aa772a96" + ), + "latest_visible_harbor_110_repair_run_id": "4108", + "latest_visible_harbor_110_repair_run_status": "Waiting", + "latest_visible_harbor_110_repair_no_matching_runner_label": ( + "awoooi-host" + ), + "latest_visible_harbor_110_repair_waiting": True, + "latest_visible_harbor_110_repair_blocked": True, + "harbor_110_repair_jobs_stale_or_mismatched": True, + "harbor_110_repair_jobs_unexpected_names": ["ai-code-review"], + "harbor_110_repair_jobs_labels": ["ubuntu-latest"], + "harbor_110_repair_jobs_runner_names": ["wooo-runner"], + }, + "rollups": { + "harbor_110_repair_blocked": True, + "harbor_110_repair_waiting": True, + "harbor_110_repair_no_matching_runner_label": "awoooi-host", + "harbor_110_repair_jobs_stale_or_mismatched": True, + }, + } diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index c50576a5b..1222e06ef 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -50878,3 +50878,27 @@ production browser smoke: **下一步**： - rebase 最新 Gitea main 後 commit / push；若 CD 仍卡 Harbor / 110，P0 runtime 下一步仍是依 phase 在 110 local console / root shell 執行 check-mode / bounded repair，並提交 receipt 驗證 phase readback。 + +## 2026-06-30 — 23:08 Harbor recovery receipt Gitea queue metadata readback + +**完成內容**： +- `harbor-registry-controlled-recovery-receipt` 新增 `gitea_actions_queue_readback` metadata-only ingestion，將 public Gitea queue readback 中的 CD run、Harbor local repair run、`awoooi-host` no-matching runner、stale / mismatched jobs 與 operation boundary 轉成 receipt-side readback。 +- 新增 queue blocker 分類：`gitea_queue_harbor_110_repair_no_matching_runner`、`gitea_queue_harbor_110_repair_jobs_stale_or_mismatched`、`gitea_queue_harbor_110_repair_blocked`、`gitea_queue_readback_boundary_violation`。 +- registry verifier 已綠但 Gitea queue 仍卡住時，receipt status 會轉成 `harbor_registry_recovery_receipt_verified_waiting_gitea_queue_clearance`，避免把 source / registry receipt 誤報成 CD 可完成。 +- queue readback 不回傳 `top_visible_runs`、raw log、HTML 或 secret；只保留 run id、狀態、runner label、job name / label / runner name 等脫敏 metadata，供 KM / RAG / MCP / PlayBook / AI Agent 回寫。 + +**本地驗證結果**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost:5432/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py -q`：`4 passed`。 +- `ruff check`、`py_compile`：通過。 + +**live truth**： +- 23:08 前一輪 live readback：CD `#4110` Running / `5d10a9d11 feat(agent): add harbor recovery phase receipts`；Harbor repair `#4108` Waiting，no matching label `awoooi-host`；jobs API 讀到 stale / mismatched `ai-code-review` / `ubuntu-latest`，不可視為 Harbor repair 成功。 +- `https://registry.wooo.work/v2/`、`https://harbor.wooo.work/api/v2.0/health`、`http://192.168.0.110:5000/v2/` 仍回 502。 + +**仍維持**： +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有讀 `.runner` 內容。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB restart，沒有 workflow_dispatch，沒有 runtime write。 + +**下一步**： +- 跑 focused suite / guards 後 commit / push；runtime 仍卡 Harbor / 110 時，P0 下一步仍是恢復 `awoooi-host` controlled lane 或在 110 local console / root shell 依 phase 執行 check-mode / bounded repair，再提交 queue + registry + deploy marker receipt。