From f4ca49c51657f0ca5953b4ce68872929cc4432a4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 00:59:03 +0800 Subject: [PATCH] fix(ops): classify 110 recovery receipt blockers --- ...or_registry_controlled_recovery_receipt.py | 139 +++- ...or_registry_controlled_recovery_receipt.py | 130 ++- docs/LOGBOOK.md | 23 + ...recovery-readback-2026-07-02.snapshot.json | 752 ++++++++++++++++++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 6 +- docs/runbooks/REBOOT-RECOVERY-SOP.md | 24 +- 6 files changed, 1042 insertions(+), 32 deletions(-) create mode 100644 docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json diff --git a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py index 3d20f66c9..7e66c21ee 100644 --- a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py @@ -435,22 +435,32 @@ def _control_path_readiness( ) -> dict[str, Any]: public_registry_ready = bool(verifier["public_registry_v2_ready"]) internal_registry_ready = bool(verifier["internal_registry_v2_ready"]) + command_path_ready = bool(ssh_diagnosis["wooo_command_path_ready"]) + harbor_110_repair_queue_superseded = ( + command_path_ready and public_registry_ready and internal_registry_ready + ) queue_no_matching_runner = bool( gitea_queue["harbor_110_repair_no_matching_runner"] - ) - queue_jobs_stale = bool(gitea_queue["harbor_110_repair_jobs_stale_or_mismatched"]) + ) and not harbor_110_repair_queue_superseded + queue_jobs_stale = bool( + gitea_queue["harbor_110_repair_jobs_stale_or_mismatched"] + ) and not harbor_110_repair_queue_superseded queue_jobs_cross_workflow = bool( gitea_queue["harbor_110_repair_jobs_cross_workflow_mismatch"] - ) + ) and not harbor_110_repair_queue_superseded queue_harbor_running_jobs_api_stale = bool( gitea_queue["harbor_110_repair_visible_running_jobs_api_stale"] - ) + ) and not harbor_110_repair_queue_superseded current_cd_waiting_behind_harbor_running = bool( gitea_queue["current_cd_waiting_behind_harbor_110_repair_running"] - ) - queue_remote_control_unavailable = bool( + ) and not harbor_110_repair_queue_superseded + queue_remote_control_unavailable_raw = bool( gitea_queue["harbor_110_repair_remote_control_channel_unavailable"] ) + queue_remote_control_unavailable = ( + queue_remote_control_unavailable_raw + and not harbor_110_repair_queue_superseded + ) cd_harbor_repair_requires_110_lane = bool( gitea_queue["current_cd_harbor_repair_requires_110_controlled_lane"] ) @@ -469,7 +479,6 @@ def _control_path_readiness( non110_runner_not_ready = bool( non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"] ) - command_path_ready = bool(ssh_diagnosis["wooo_command_path_ready"]) runner_timeout = bool( ssh_diagnosis["runner_systemctl_show_timeout_effective_seen"] ) @@ -590,6 +599,9 @@ def _control_path_readiness( "harbor_110_repair_remote_control_channel_unavailable": ( queue_remote_control_unavailable ), + "harbor_110_repair_remote_control_channel_unavailable_raw": ( + queue_remote_control_unavailable_raw + ), "harbor_110_repair_failure_classifier": gitea_queue[ "harbor_110_repair_failure_classifier" ], @@ -719,6 +731,8 @@ def _control_path_readiness_status( return "blocked_awoooi_host_runner_queue_unavailable" if node_high_load: return "blocked_110_node_high_load" + if any(item.startswith("gitea_queue_cd_jobs_") for item in signal_ids): + return "blocked_gitea_cd_jobs_readback_stale_or_mismatched" if not registry_v2_ready: return "blocked_registry_v2_verifier_not_green" return "blocked_control_path_evidence_not_clear" @@ -739,6 +753,8 @@ def _control_path_safe_next_action(*, status: str) -> str: return "rerun_non_secret_110_ssh_and_runner_control_path_diagnosis" if status == "blocked_registry_v2_verifier_not_green": return "rerun_public_and_internal_registry_v2_verifier_before_cd_retry" + if status == "blocked_gitea_cd_jobs_readback_stale_or_mismatched": + return "rerun_public_gitea_queue_and_deploy_marker_readback_before_cd_retry" return "normalize_control_path_evidence_then_retry_readback" @@ -1240,6 +1256,25 @@ def _parse_controlled_cd_lane_readiness_output(output: str) -> dict[str, Any]: def _parse_non110_runner_readiness_output(output: str) -> dict[str, Any]: fields = _parse_key_values(output) marker_seen = "AWOOOI_NON110_RUNNER_READY=" in output + if not marker_seen: + return { + "receipt_seen": False, + "non110_runner_ready": False, + "ready_config_count": 0, + "ready_binary_count": 0, + "ready_registration_count": 0, + "ready_service_count": 0, + "ready_active_service_count": 0, + "ready_autostart_path_count": 0, + "warning_count": 0, + "blocker_count": 0, + "blockers": [], + "safe_next_step": "", + "raw_runner_registration_read": False, + "runner_token_read": False, + "metadata_only": True, + "raw_output_returned": False, + } blockers = _prefixed_blockers( output, prefix="non110_runner_readiness:", @@ -2086,11 +2121,85 @@ def _active_blockers( blockers.append("public_registry_v2_verifier_not_green") if not verifier["internal_registry_v2_ready"]: blockers.append("internal_registry_v2_verifier_not_green") - blockers.extend(_strings(gitea_queue.get("blockers"))) + blockers.extend( + _effective_gitea_queue_blockers( + ssh_diagnosis=ssh_diagnosis, + ssh_local=ssh_local, + watchdog_check=watchdog_check, + controlled_cd_lane=controlled_cd_lane, + verifier=verifier, + gitea_queue=gitea_queue, + ) + ) blockers.extend(_strings(deploy_marker.get("blockers"))) return _unique_strings(blockers) +def _effective_gitea_queue_blockers( + *, + ssh_diagnosis: dict[str, Any], + ssh_local: dict[str, Any], + watchdog_check: dict[str, Any], + controlled_cd_lane: dict[str, Any], + verifier: dict[str, Any], + gitea_queue: dict[str, Any], +) -> list[str]: + blockers = _strings(gitea_queue.get("blockers")) + superseded = _gitea_queue_blockers_superseded_by_local_receipt( + ssh_diagnosis=ssh_diagnosis, + ssh_local=ssh_local, + watchdog_check=watchdog_check, + controlled_cd_lane=controlled_cd_lane, + verifier=verifier, + ) + if not superseded: + return blockers + return [blocker for blocker in blockers if blocker not in superseded] + + +def _gitea_queue_blockers_superseded_by_local_receipt( + *, + ssh_diagnosis: dict[str, Any], + ssh_local: dict[str, Any], + watchdog_check: dict[str, Any], + controlled_cd_lane: dict[str, Any], + verifier: dict[str, Any], +) -> set[str]: + control_channel_ready = bool( + ssh_diagnosis["wooo_command_path_ready"] + or ssh_local["control_channel_metadata_ready"] + ) + harbor_ready = bool(watchdog_check["harbor_ready"] or verifier["registry_v2_ready"]) + if not control_channel_ready or not harbor_ready: + return set() + + superseded = { + "gitea_queue_harbor_110_remote_control_channel_unavailable", + "gitea_queue_harbor_110_repair_blocked", + "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale", + "gitea_queue_current_cd_waiting_behind_harbor_110_repair_running", + } + if verifier["registry_v2_ready"]: + superseded.update( + { + "gitea_queue_current_cd_harbor_retrying_unavailable", + "gitea_queue_harbor_110_remote_local_registry_v2_unavailable", + "gitea_queue_harbor_public_registry_v2_unavailable_after_remote_repair", + } + ) + if controlled_cd_lane["receipt_seen"]: + superseded.update( + { + "gitea_queue_harbor_110_repair_no_matching_runner", + "gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch", + "gitea_queue_harbor_110_repair_jobs_stale_or_mismatched", + "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host", + "gitea_queue_cd_harbor_repair_requires_110_controlled_lane", + } + ) + return superseded + + def _ssh_local_metadata_blockers(ssh_local: dict[str, Any]) -> list[str]: blockers: list[str] = [] if not ssh_local["sshd_config_syntax_ok"]: @@ -2129,10 +2238,16 @@ def _status( ) -> str: if verifier["registry_v2_ready"] and not active_blockers: return "harbor_registry_recovery_receipt_verified" + if controlled_cd_lane["receipt_seen"] and not controlled_cd_lane[ + "controlled_cd_lane_ready" + ]: + return "controlled_cd_lane_readiness_receipt_blocked" + if non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]: + return "non110_runner_readiness_receipt_blocked" if ( verifier["registry_v2_ready"] and gitea_queue["receipt_seen"] - and gitea_queue["blocker_count"] > 0 + and any(blocker.startswith("gitea_queue_") for blocker in active_blockers) ): return "harbor_registry_recovery_receipt_verified_waiting_gitea_queue_clearance" if ( @@ -2141,12 +2256,6 @@ def _status( and deploy_marker["blocker_count"] > 0 ): return "harbor_registry_recovery_receipt_verified_waiting_deploy_marker_readback" - if controlled_cd_lane["receipt_seen"] and not controlled_cd_lane[ - "controlled_cd_lane_ready" - ]: - return "controlled_cd_lane_readiness_receipt_blocked" - if non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]: - return "non110_runner_readiness_receipt_blocked" if watchdog_repair["receipt_seen"]: return "harbor_registry_repair_receipt_waiting_registry_v2_verifier" if watchdog_check["receipt_seen"] and watchdog_check["harbor_ready"]: diff --git a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py index 2d2510cc7..61622f7b6 100644 --- a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py @@ -297,19 +297,9 @@ def test_harbor_recovery_receipt_accepts_ssh_publickey_diagnosis() -> None: def test_harbor_recovery_receipt_command_path_ready_overrides_stale_timeout() -> None: - diagnosis_output = _ssh_publickey_diagnosis_output().replace( - "SSH_AUTH user=wooo mode=password_disabled rc=255 classification=permission_denied", - ( - "SSH_COMMAND_PATH user=wooo rc=0 classification=command_path_ready " - "marker_seen=true remote_user_match=true\n" - "SSH_AUTH user=wooo mode=password_disabled rc=255 " - "classification=permission_denied" - ), - ) - payload = validate_harbor_registry_controlled_recovery_receipt( { - "ssh_publickey_diagnosis_output": diagnosis_output, + "ssh_publickey_diagnosis_output": _ssh_publickey_command_path_ready_output(), "watchdog_check_output": _watchdog_check_output(ready=True, status=401), "public_registry_v2_http_status": 401, "internal_registry_v2_http_status": 401, @@ -348,6 +338,66 @@ def test_harbor_recovery_receipt_command_path_ready_overrides_stale_timeout() -> ) +def test_harbor_recovery_receipt_local_receipt_supersedes_stale_gitea_queue() -> None: + payload = validate_harbor_registry_controlled_recovery_receipt( + { + "ssh_publickey_diagnosis_output": _ssh_publickey_command_path_ready_output(), + "local_console_recovery_output": "\n".join( + [ + _watchdog_check_output(ready=True, status=401), + _controlled_cd_lane_blocked_output(), + ] + ), + "public_registry_v2_http_status": 401, + "internal_registry_v2_http_status": 401, + "gitea_actions_queue_readback": _gitea_queue_remote_control_timeout(), + } + ) + + assert payload["status"] == "controlled_cd_lane_readiness_receipt_blocked" + assert payload["safe_next_step"] == ( + "fix_controlled_cd_lane_guardrail_blockers_then_rerun_readiness_verifier" + ) + assert "controlled_cd_lane_readiness:controlled_cd_lane_registration_missing" in ( + payload["active_blockers"] + ) + assert "controlled_cd_lane_readiness:controlled_cd_lane_service_not_active" in ( + payload["active_blockers"] + ) + assert not any( + blocker.startswith("gitea_queue_harbor_110") + for blocker in payload["active_blockers"] + ) + assert not any( + blocker.startswith("non110_runner_readiness:") + for blocker in payload["active_blockers"] + ) + assert payload["readback"]["non110_runner_readiness"]["receipt_seen"] is False + queue = payload["readback"]["gitea_actions_queue"] + assert queue["harbor_110_repair_remote_control_channel_unavailable"] is True + assert queue["blocker_count"] == 2 + readiness = payload["readback"]["control_path_readiness"] + assert readiness["status"] == "ready" + assert readiness["ssh_command_path_ready"] is True + assert readiness["harbor_110_repair_remote_control_channel_unavailable"] is False + assert ( + readiness["harbor_110_repair_remote_control_channel_unavailable_raw"] + is True + ) + assert readiness["harbor_110_repair_jobs_stale_or_mismatched"] is False + assert not any( + signal.startswith("gitea_queue_harbor_110") + for signal in readiness["signal_ids"] + ) + assert payload["rollups"]["gitea_queue_blocker_count"] == 2 + assert ( + payload["rollups"][ + "gitea_queue_harbor_110_remote_control_channel_unavailable" + ] + is True + ) + + def test_harbor_recovery_receipt_classifies_publickey_offer_timeout() -> None: diagnosis_output = _ssh_publickey_diagnosis_output().replace( "rc=124 classification=server_accepts_key_then_timeout", @@ -893,6 +943,32 @@ def test_harbor_recovery_receipt_surfaces_cd_jobs_payload_classifier() -> None: ] +def test_harbor_recovery_receipt_classifies_cd_jobs_only_readback_mismatch() -> None: + payload = validate_harbor_registry_controlled_recovery_receipt( + { + "watchdog_check_output": _watchdog_check_output( + ready=True, + status=401, + ), + "public_registry_v2_http_status": 401, + "internal_registry_v2_http_status": 401, + "gitea_actions_queue_readback": _gitea_queue_cd_jobs_head_sha_mismatch_only(), + } + ) + + readiness = payload["readback"]["control_path_readiness"] + assert readiness["status"] == ( + "blocked_gitea_cd_jobs_readback_stale_or_mismatched" + ) + assert readiness["safe_next_action"] == ( + "rerun_public_gitea_queue_and_deploy_marker_readback_before_cd_retry" + ) + assert readiness["signal_ids"] == [ + "gitea_queue_cd_jobs_head_sha_mismatch", + "gitea_queue_cd_jobs_stale_or_mismatched", + ] + + def test_harbor_recovery_receipt_surfaces_current_cd_runner_readiness() -> None: payload = validate_harbor_registry_controlled_recovery_receipt( { @@ -1157,6 +1233,18 @@ INTERPRETATION=server_accepts_key_then_timeout_on_wooo_means_check_110_session_p """ +def _ssh_publickey_command_path_ready_output() -> str: + return _ssh_publickey_diagnosis_output().replace( + "SSH_AUTH user=wooo mode=password_disabled rc=255 classification=permission_denied", + ( + "SSH_COMMAND_PATH user=wooo rc=0 classification=command_path_ready " + "marker_seen=true remote_user_match=true\n" + "SSH_AUTH user=wooo mode=password_disabled rc=255 " + "classification=permission_denied" + ), + ) + + def _watchdog_check_output(*, ready: bool, status: int) -> str: ready_text = "true" if ready else "false" return f""" @@ -1412,6 +1500,26 @@ def _gitea_queue_cd_jobs_head_sha_mismatch() -> dict: return payload +def _gitea_queue_cd_jobs_head_sha_mismatch_only() -> dict: + payload = _gitea_queue_cd_jobs_head_sha_mismatch() + payload["readback"].update( + { + "latest_visible_harbor_110_repair_no_matching_runner_label": "", + "harbor_110_repair_jobs_stale_or_mismatched": False, + "harbor_110_repair_jobs_unexpected_names": [], + "harbor_110_repair_jobs_labels": [], + "harbor_110_repair_jobs_runner_names": [], + } + ) + payload["rollups"].update( + { + "harbor_110_repair_no_matching_runner_label": "", + "harbor_110_repair_jobs_stale_or_mismatched": False, + } + ) + return payload + + def _gitea_queue_current_cd_waiting_non110() -> dict: payload = _gitea_queue_cd_jobs_head_sha_mismatch() payload["status"] = "blocked_current_cd_workflow_waiting_for_runner_or_queue" diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 7cb6a4019..2c21772b1 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -51934,6 +51934,29 @@ production browser smoke: - 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 - 沒有重啟主機，沒有 Docker / Nginx / K3s / DB restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune。 +## 2026-07-02 — 110 control-path / Harbor recovery receipt 與 stale queue 收斂 + +**完成內容**： +- 修正 `apps/api/src/services/harbor_registry_controlled_recovery_receipt.py`：當同一輪本地 read-only evidence 已證明 `wooo` command path ready 且 public/internal registry `/v2/` ready，Gitea Harbor repair 的 historical SSH / remote-control / stale jobs blocker 不再進入 active blockers。 +- 補強 `control_path_readiness` 狀態：只有 CD jobs head-SHA / stale readback mismatch 時，回 `blocked_gitea_cd_jobs_readback_stale_or_mismatched`，不再落到泛用 `blocked_control_path_evidence_not_clear`。 +- 修正 non110 parser：沒有 `AWOOOI_NON110_RUNNER_READY` marker 時，不得從 combined local-console output 的 110 `BLOCKER` 行推導 non110 runner blocker。 +- 新增回歸測試，覆蓋 local command-path receipt 覆蓋 stale Gitea queue、combined local-console marker gate、CD jobs-only mismatch classifier。 +- 產出 live receipt：`docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json`。readback 顯示 public/internal registry `/v2/` 均為 `401`，Gitea latest visible CD `#4335` 為 `Success`，舊 Harbor repair failure 已是 historical after latest CD success；active blockers 剩 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale mismatch。 +- 更新 `docs/runbooks/FULL-STACK-COLD-START-SOP.md` v1.91 與 `docs/runbooks/REBOOT-RECOVERY-SOP.md` v5.1，把 2026-07-02 receipt 判讀規則沉澱進 SOP。 + +**驗證結果**： +- `DATABASE_URL=postgresql://u:p@localhost:5432/test python3.11 -m pytest apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py -q`：`26 passed`。 +- live receipt command return code：`ssh_publickey_diagnosis_rc=0`、`local_console_recovery_check_rc=0`、`public_gitea_queue_readback_rc=0`。 +- live receipt status：`controlled_cd_lane_readiness_receipt_blocked`；`safe_next_step=fix_controlled_cd_lane_guardrail_blockers_then_rerun_readiness_verifier`；non110 runner receipt `false`，沒有 non110 誤報。 + +**仍維持**： +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有讀 `.runner` 內容。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune。 + +**下一步**： +- 修復 110 controlled CD lane 的 config / binary / registration / service guardrail，並重跑 readiness verifier、public Gitea queue readback、deploy marker readback。runner registration 必須走不列印 token / 不讀 `.runner` 內容的 token-safe path。 + ## 2026-07-02 — 00:17 P0 2026-07-01 post-reboot runtime recovery GREEN 與 SLO miss 留證 **完成內容**： diff --git a/docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json b/docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json new file mode 100644 index 000000000..54cfe47b0 --- /dev/null +++ b/docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json @@ -0,0 +1,752 @@ +{ + "command_receipts": { + "internal_registry_v2_http_status": "401", + "local_console_recovery_check_rc": 0, + "public_gitea_queue_readback_rc": 0, + "public_registry_v2_http_status": "401", + "ssh_publickey_diagnosis_rc": 0 + }, + "generated_at": "2026-07-01T16:56:00.696529Z", + "operation_boundaries": { + "docker_daemon_restart_performed": false, + "github_api_used": false, + "host_reboot_performed": false, + "raw_output_returned": false, + "raw_runner_registration_read": false, + "runner_token_read": false, + "secret_value_read": false, + "ssh_used": true, + "ssh_write_performed": false, + "workflow_dispatch_performed": false + }, + "queue_readback": { + "harbor_110_repair_failure_classifier": "harbor_110_remote_ssh_publickey_auth_stalled", + "harbor_110_repair_historical_after_latest_cd_success": true, + "harbor_110_repair_run_status": "Failure", + "latest_visible_cd_run_commit_sha": "8cc96973f7c73368d6544c85f9e42c5f3a144c14", + "latest_visible_cd_run_id": "4335", + "latest_visible_cd_run_status": "Success", + "status": "no_matching_runner_not_visible" + }, + "receipt": { + "accepted": false, + "active_blocker_count": 10, + "active_blockers": [ + "controlled_cd_lane_readiness:controlled_cd_lane_config_missing", + "controlled_cd_lane_readiness:controlled_cd_lane_binary_not_ready", + "controlled_cd_lane_readiness:controlled_cd_lane_registration_missing", + "controlled_cd_lane_readiness:controlled_cd_lane_service_limits_missing", + "controlled_cd_lane_readiness:controlled_cd_lane_service_target_mismatch", + "controlled_cd_lane_readiness:controlled_cd_lane_service_registration_condition_missing", + "controlled_cd_lane_readiness:controlled_cd_lane_service_not_active", + "controlled_cd_lane_readiness:active_action_containers_present:1", + "gitea_queue_cd_jobs_head_sha_mismatch", + "gitea_queue_cd_jobs_stale_or_mismatched" + ], + "controlled_apply_policy": { + "allowed_scope": [ + "validate_non_secret_ssh_publickey_diagnosis_receipt", + "validate_non_secret_ssh_local_repair_receipt", + "validate_harbor_watchdog_check_receipt", + "validate_harbor_watchdog_repair_once_receipt", + "validate_110_controlled_cd_lane_readiness_receipt", + "validate_non110_runner_readiness_receipt", + "validate_public_and_internal_registry_v2_verifier", + "km_rag_mcp_playbook_metadata_writeback", + "retry_gitea_cd_after_registry_v2_green" + ], + "break_glass_required": false, + "current_apply_allowed": false, + "current_apply_blocker": "controlled_cd_lane_readiness_required_for_awoooi_host_queue", + "forbidden_scope": [ + "secret_read", + "docker_daemon_restart", + "host_reboot", + "node_drain", + "nginx_reload_or_restart", + "database_restore_or_prune", + "workflow_dispatch_from_receipt_validator", + "force_push_or_repo_ref_mutation", + "registry_provider_switch" + ], + "manual_end_state": false, + "risk_level": "high" + }, + "input_redaction": { + "controlled_cd_lane_readiness_output": { + "byte_count": 0, + "line_count": 0, + "provided": false + }, + "deploy_marker_readback": { + "metadata_only": true, + "provided": false, + "raw_output_returned": false + }, + "gitea_actions_queue_readback": { + "metadata_only": true, + "provided": true, + "raw_output_returned": false, + "top_visible_runs_returned": false + }, + "local_console_recovery_output": { + "byte_count": 5585, + "line_count": 112, + "provided": true + }, + "non110_runner_readiness_output": { + "byte_count": 0, + "line_count": 0, + "provided": false + }, + "raw_output_returned": false, + "ssh_local_repair_output": { + "byte_count": 0, + "line_count": 0, + "provided": false + }, + "ssh_publickey_diagnosis_output": { + "byte_count": 2191, + "line_count": 34, + "provided": true + }, + "watchdog_check_output": { + "byte_count": 0, + "line_count": 0, + "provided": false + }, + "watchdog_repair_output": { + "byte_count": 0, + "line_count": 0, + "provided": false + } + }, + "learning_writeback_contracts": [ + { + "raw_log_allowed": false, + "receipt_key": "harbor_registry_recovery_receipt::controlled_cd_lane_readiness_receipt_blocked::timeline_event", + "secret_value_allowed": false, + "target": "timeline_event", + "writeback_mode": "metadata_only" + }, + { + "raw_log_allowed": false, + "receipt_key": "harbor_registry_recovery_receipt::controlled_cd_lane_readiness_receipt_blocked::knowledge_entry", + "secret_value_allowed": false, + "target": "knowledge_entry", + "writeback_mode": "metadata_only" + }, + { + "raw_log_allowed": false, + "receipt_key": "harbor_registry_recovery_receipt::controlled_cd_lane_readiness_receipt_blocked::rag_embedding", + "secret_value_allowed": false, + "target": "rag_embedding", + "writeback_mode": "metadata_only" + }, + { + "raw_log_allowed": false, + "receipt_key": "harbor_registry_recovery_receipt::controlled_cd_lane_readiness_receipt_blocked::mcp_tool_registry_signal", + "secret_value_allowed": false, + "target": "mcp_tool_registry_signal", + "writeback_mode": "metadata_only" + }, + { + "raw_log_allowed": false, + "receipt_key": "harbor_registry_recovery_receipt::controlled_cd_lane_readiness_receipt_blocked::playbook_trust_score", + "secret_value_allowed": false, + "target": "playbook_trust_score", + "writeback_mode": "metadata_only" + }, + { + "raw_log_allowed": false, + "receipt_key": "harbor_registry_recovery_receipt::controlled_cd_lane_readiness_receipt_blocked::controlled_recovery_report", + "secret_value_allowed": false, + "target": "controlled_recovery_report", + "writeback_mode": "metadata_only" + } + ], + "local_console_phase_readback": { + "blocked_phase_count": 1, + "completed_phase_count": 5, + "metadata_only": true, + "phase_count": 6, + "phase_ids": [ + "diagnose_ssh_publickey", + "preflight_control_path_and_harbor", + "repair_ssh_metadata_if_check_confirms_metadata_drift", + "repair_harbor_once_if_v2_still_502", + "verify_controlled_cd_lane", + "verify_non110_runner_lane" + ], + "phases": [ + { + "evidence_key": "ssh_publickey_diagnosis", + "phase_id": "diagnose_ssh_publickey", + "raw_output_returned": false, + "status": "ready" + }, + { + "evidence_key": "watchdog_check", + "phase_id": "preflight_control_path_and_harbor", + "raw_output_returned": false, + "status": "ready" + }, + { + "evidence_key": "ssh_local_repair", + "phase_id": "repair_ssh_metadata_if_check_confirms_metadata_drift", + "raw_output_returned": false, + "status": "ready" + }, + { + "evidence_key": "watchdog_repair", + "phase_id": "repair_harbor_once_if_v2_still_502", + "raw_output_returned": false, + "status": "skipped_not_required" + }, + { + "evidence_key": "controlled_cd_lane_readiness", + "phase_id": "verify_controlled_cd_lane", + "raw_output_returned": false, + "status": "blocked_controlled_cd_lane_readiness_receipt_not_ready" + }, + { + "evidence_key": "non110_runner_readiness", + "phase_id": "verify_non110_runner_lane", + "raw_output_returned": false, + "status": "skipped_not_required" + } + ], + "raw_output_returned": false + }, + "operation_boundaries": { + "database_write_or_restore_performed": false, + "docker_command_performed": false, + "docker_daemon_restart_performed": false, + "docker_restart_performed": false, + "github_api_used": false, + "host_reboot_performed": false, + "nginx_reload_or_restart_performed": false, + "node_drain_performed": false, + "raw_output_returned": false, + "runtime_write_allowed_by_this_validator": false, + "secret_value_collection_allowed": false, + "ssh_used": false, + "workflow_trigger_performed": false + }, + "priority": "P0-006", + "readback": { + "control_path_readiness": { + "awoooi_host_runner_unavailable": false, + "blocker_count": 2, + "cd_harbor_repair_blocked_by_no_matching_awoooi_host": false, + "cd_harbor_repair_lane_classifier": "", + "cd_harbor_repair_requires_110_controlled_lane": false, + "cd_jobs_head_sha_mismatch": true, + "cd_jobs_stale_or_mismatched": true, + "controlled_profile_no_matching_runner_label_count": 0, + "current_cd_no_matching_runner_label": "", + "current_cd_waiting_behind_harbor_110_repair_running": false, + "current_cd_waiting_for_runner_or_queue": false, + "gitea_queue_readback_seen": true, + "harbor_110_repair_bounded_ssh_timeout_seen": false, + "harbor_110_repair_failure_classifier": "harbor_110_remote_ssh_publickey_auth_stalled", + "harbor_110_repair_jobs_cross_workflow_mismatch": false, + "harbor_110_repair_jobs_stale_or_mismatched": false, + "harbor_110_repair_no_matching_runner": false, + "harbor_110_repair_no_matching_runner_label": "", + "harbor_110_repair_remote_control_channel_unavailable": false, + "harbor_110_repair_remote_control_channel_unavailable_raw": true, + "harbor_110_repair_remote_ssh_reachable": true, + "harbor_110_repair_visible_running_jobs_api_stale": false, + "metadata_only": true, + "node_high_load": false, + "node_load1_per_cpu": 0.41, + "node_load_classifier": "load_not_high", + "non110_runner_blocker_count": 0, + "non110_runner_blockers": [], + "non110_runner_readiness_receipt_seen": false, + "non110_runner_ready": false, + "non110_runner_safe_next_step": "", + "non110_runner_unavailable": false, + "primary_blocker": "gitea_queue_cd_jobs_head_sha_mismatch", + "raw_output_returned": false, + "registry_v2_internal_http_status": 401, + "registry_v2_internal_ready": true, + "registry_v2_public_http_status": 401, + "registry_v2_public_ready": true, + "registry_v2_ready": true, + "runner_systemctl_show_timeout": false, + "runner_systemctl_show_timeout_raw": false, + "safe_next_action": "rerun_public_gitea_queue_and_deploy_marker_readback_before_cd_retry", + "signal_ids": [ + "gitea_queue_cd_jobs_head_sha_mismatch", + "gitea_queue_cd_jobs_stale_or_mismatched" + ], + "ssh_command_path_classification": "command_path_ready", + "ssh_command_path_ready": true, + "ssh_diagnosis_receipt_seen": true, + "ssh_port_tcp_open": true, + "ssh_publickey_offer_timeout": false, + "ssh_publickey_offer_timeout_raw": false, + "ssh_server_accepts_key_then_timeout": false, + "ssh_server_accepts_key_then_timeout_raw": true, + "status": "blocked_gitea_cd_jobs_readback_stale_or_mismatched" + }, + "controlled_cd_lane_readiness": { + "binary_ready": false, + "blocker_count": 8, + "blockers": [ + "controlled_cd_lane_readiness:controlled_cd_lane_config_missing", + "controlled_cd_lane_readiness:controlled_cd_lane_binary_not_ready", + "controlled_cd_lane_readiness:controlled_cd_lane_registration_missing", + "controlled_cd_lane_readiness:controlled_cd_lane_service_limits_missing", + "controlled_cd_lane_readiness:controlled_cd_lane_service_target_mismatch", + "controlled_cd_lane_readiness:controlled_cd_lane_service_registration_condition_missing", + "controlled_cd_lane_readiness:controlled_cd_lane_service_not_active", + "controlled_cd_lane_readiness:active_action_containers_present:1" + ], + "config_ready": false, + "controlled_cd_lane_ready": false, + "legacy_failclosed": true, + "metadata_only": true, + "primary_lane_failclosed": true, + "raw_output_returned": false, + "receipt_seen": true, + "registration_ready": false, + "safe_next_step": "restore_or_register_awoooi_cd_lane_drain_registration_without_printing_token_then_rerun_this_verifier", + "service_ready": false, + "warning_count": 0 + }, + "deploy_marker": { + "blocker_count": 0, + "blockers": [], + "deploy_marker_commit_sha": "", + "deploy_marker_verified": false, + "expected_commit_sha": "", + "latest_visible_cd_run_id": "", + "latest_visible_cd_run_status": "", + "metadata_only": true, + "operation_boundary_violation": false, + "production_image_matches_expected": false, + "production_image_tag_sha": "", + "production_preflight_http_status": null, + "production_readback_http_green": false, + "production_workbench_http_status": null, + "raw_output_returned": false, + "receipt_seen": false, + "schema_version": "", + "status": "not_provided" + }, + "gitea_actions_queue": { + "blocker_count": 4, + "blockers": [ + "gitea_queue_cd_jobs_head_sha_mismatch", + "gitea_queue_cd_jobs_stale_or_mismatched", + "gitea_queue_harbor_110_remote_control_channel_unavailable", + "gitea_queue_harbor_110_repair_jobs_stale_or_mismatched" + ], + "cd_run_jobs_expected_head_sha": "8cc96973f7c73368d6544c85f9e42c5f3a144c14", + "cd_run_jobs_expected_run_id": "4335", + "cd_run_jobs_head_sha_mismatch": true, + "cd_run_jobs_head_shas": [ + "45c2b8ebe642422be0dd3e2e438c60e38f97509f" + ], + "cd_run_jobs_payload_classifier": "cd_jobs_api_head_sha_mismatch_for_visible_cd_run", + "cd_run_jobs_run_id_mismatch": false, + "cd_run_jobs_run_ids": [ + "4335" + ], + "cd_run_jobs_stale_or_mismatched": true, + "controlled_profile_no_matching_runner_label_count": 0, + "controlled_profile_no_matching_runner_labels": {}, + "current_cd_harbor_latest_registry_v2_status": "", + "current_cd_harbor_repair_blocked_by_no_matching_awoooi_host": false, + "current_cd_harbor_repair_lane_classifier": "", + "current_cd_harbor_repair_requires_110_controlled_lane": false, + "current_cd_harbor_retrying_unavailable": false, + "current_cd_inflight_classifier": "", + "current_cd_no_matching_runner_label": "", + "current_cd_waiting_behind_harbor_110_repair_running": false, + "current_cd_waiting_for_runner_or_queue": false, + "harbor_110_repair_blocked": false, + "harbor_110_repair_bounded_ssh_timeout_seen": false, + "harbor_110_repair_failure_classifier": "harbor_110_remote_ssh_publickey_auth_stalled", + "harbor_110_repair_jobs_cross_workflow_mismatch": false, + "harbor_110_repair_jobs_expected_names": [ + "harbor-110-local-repair", + "workflow-shape" + ], + "harbor_110_repair_jobs_labels": [ + "ubuntu-latest" + ], + "harbor_110_repair_jobs_payload_classifier": "unexpected_harbor_110_repair_job_names", + "harbor_110_repair_jobs_runner_names": [ + "wooo-runner" + ], + "harbor_110_repair_jobs_stale_or_mismatched": true, + "harbor_110_repair_jobs_unexpected_names": [ + "ai-code-review" + ], + "harbor_110_repair_local_registry_v2_status": "", + "harbor_110_repair_no_matching_runner": false, + "harbor_110_repair_no_matching_runner_label": "", + "harbor_110_repair_public_registry_v2_status": "", + "harbor_110_repair_remote_control_channel_unavailable": true, + "harbor_110_repair_remote_ssh_reachable": true, + "harbor_110_repair_verified": true, + "harbor_110_repair_visible_running_jobs_api_stale": false, + "harbor_110_repair_waiting": false, + "latest_visible_cd_run_commit_sha": "8cc96973f7c73368d6544c85f9e42c5f3a144c14", + "latest_visible_cd_run_id": "4335", + "latest_visible_cd_run_status": "Success", + "latest_visible_cd_run_waiting": false, + "latest_visible_harbor_110_repair_run_id": "4312", + "latest_visible_harbor_110_repair_run_status": "Failure", + "metadata_only": true, + "normalized_classifier_field_count": 4, + "normalized_classifier_field_ids": [ + "cd_run_jobs_payload_classifier", + "harbor_110_repair_jobs_payload_classifier", + "latest_visible_harbor_110_repair_no_matching_runner_label", + "harbor_110_repair_failure_classifier" + ], + "normalized_classifier_fields": [ + { + "blockers": [ + "gitea_queue_cd_jobs_head_sha_mismatch", + "gitea_queue_cd_jobs_stale_or_mismatched" + ], + "field_id": "cd_run_jobs_payload_classifier", + "metadata_only": true, + "raw_output_returned": false, + "value": "cd_jobs_api_head_sha_mismatch_for_visible_cd_run" + }, + { + "blockers": [ + "gitea_queue_harbor_110_repair_jobs_stale_or_mismatched" + ], + "field_id": "harbor_110_repair_jobs_payload_classifier", + "metadata_only": true, + "raw_output_returned": false, + "value": "unexpected_harbor_110_repair_job_names" + }, + { + "blockers": [], + "field_id": "latest_visible_harbor_110_repair_no_matching_runner_label", + "metadata_only": true, + "raw_output_returned": false, + "value": "" + }, + { + "blockers": [ + "gitea_queue_harbor_110_remote_control_channel_unavailable" + ], + "field_id": "harbor_110_repair_failure_classifier", + "metadata_only": true, + "raw_output_returned": false, + "value": "harbor_110_remote_ssh_publickey_auth_stalled" + } + ], + "operation_boundary_violation": false, + "raw_output_returned": false, + "receipt_seen": true, + "schema_version": "awoooi_public_gitea_actions_queue_readback_v1", + "status": "no_matching_runner_not_visible", + "top_visible_runs_returned": false + }, + "non110_runner_readiness": { + "blocker_count": 0, + "blockers": [], + "metadata_only": true, + "non110_runner_ready": false, + "raw_output_returned": false, + "raw_runner_registration_read": false, + "ready_active_service_count": 0, + "ready_autostart_path_count": 0, + "ready_binary_count": 0, + "ready_config_count": 0, + "ready_registration_count": 0, + "ready_service_count": 0, + "receipt_seen": false, + "runner_token_read": false, + "safe_next_step": "", + "warning_count": 0 + }, + "post_apply_verifier": { + "internal_registry_v2_http_status": 401, + "internal_registry_v2_ready": true, + "public_registry_v2_http_status": 401, + "public_registry_v2_ready": true, + "registry_v2_ready": true + }, + "ssh_local_repair": { + "account_metadata_ready": true, + "authorized_keys_metadata_present": true, + "control_channel_metadata_ready": true, + "mode": "check", + "permissions_applied": false, + "receipt_seen": true, + "ssh_reload_done": false, + "ssh_reload_skipped": false, + "sshd_authorized_keys_file_default": true, + "sshd_config_syntax_after_apply_ok": true, + "sshd_config_syntax_after_apply_unverified_requires_root": false, + "sshd_config_syntax_ok": true, + "sshd_config_syntax_unverified_requires_root": false, + "sshd_effective_config_available": true, + "sshd_kbdinteractiveauthentication": "no", + "sshd_maxstartups": "10:30:100", + "sshd_passwordauthentication": "yes", + "sshd_pubkeyauthentication": "yes", + "sshd_usepam": "yes", + "target_user_account_locked": false, + "target_user_exists": true, + "target_user_shell_executable": true + }, + "ssh_publickey_diagnosis": { + "auth_attempt_count": 8, + "auth_classifications": [ + { + "classification": "server_accepts_key_then_timeout", + "mode": "publickey", + "rc": 0, + "user": "wooo" + }, + { + "classification": "permission_denied", + "mode": "publickey", + "rc": 255, + "user": "root" + }, + { + "classification": "permission_denied", + "mode": "publickey", + "rc": 255, + "user": "git" + }, + { + "classification": "permission_denied", + "mode": "publickey", + "rc": 255, + "user": "ollama" + }, + { + "classification": "permission_denied", + "mode": "password_disabled", + "rc": 255, + "user": "wooo" + }, + { + "classification": "permission_denied", + "mode": "password_disabled", + "rc": 255, + "user": "root" + }, + { + "classification": "permission_denied", + "mode": "password_disabled", + "rc": 255, + "user": "git" + }, + { + "classification": "permission_denied", + "mode": "password_disabled", + "rc": 255, + "user": "ollama" + } + ], + "command_path_attempt_count": 1, + "command_path_classifications": [ + { + "classification": "command_path_ready", + "marker_seen": true, + "rc": 0, + "remote_user_match": true, + "user": "wooo" + } + ], + "diagnosis_ready": true, + "metadata_only": true, + "node_exporter_ok": true, + "node_high_load_seen": false, + "node_load1_per_cpu": 0.41, + "node_load_classifier": "load_not_high", + "permission_denied_count": 7, + "preauth_timeout_count": 0, + "publickey_offer_timeout_effective_seen": false, + "publickey_offer_timeout_seen": false, + "raw_output_returned": false, + "receipt_seen": true, + "runner_systemctl_show_timeout_effective_seen": false, + "runner_systemctl_show_timeout_seen": false, + "server_accepts_key_then_timeout_effective_seen": false, + "server_accepts_key_then_timeout_seen": true, + "ssh_banner_seen": true, + "ssh_port_tcp_open": true, + "systemd_unit_signal_count": 5, + "systemd_unit_signals": [ + { + "active_state": "inactive", + "classifier": "inactive", + "unit": "actions.runner.owenhytsai-awoooi.awoooi-110-3.service" + }, + { + "active_state": "inactive", + "classifier": "inactive", + "unit": "actions.runner.owenhytsai-awoooi.awoooi-110.service" + }, + { + "active_state": "inactive", + "classifier": "inactive", + "unit": "actions.runner.owenhytsai-wooo-aiops.wooo-110-runner-2.service" + }, + { + "active_state": "inactive", + "classifier": "inactive", + "unit": "actions.runner.owenhytsai-wooo-aiops.wooo-110-runner-3.service" + }, + { + "active_state": "inactive", + "classifier": "inactive", + "unit": "actions.runner.owenhytsai-wooo-aiops.wooo-runner-110.service" + } + ], + "target": "192.168.0.110:22", + "wooo_command_path_classification": "command_path_ready", + "wooo_command_path_ready": true, + "wooo_publickey_classification": "server_accepts_key_then_timeout" + }, + "watchdog_check": { + "check_only": true, + "docker_compose_action_performed": false, + "expected_host_ip_present": true, + "forbidden_action_seen": false, + "harbor_local_v2_http_status": 401, + "harbor_ready": true, + "host_reboot_performed": false, + "mode": "check", + "receipt_seen": true, + "repair_attempt_seen": false, + "service_restart_performed": false + }, + "watchdog_repair": { + "check_only": true, + "docker_compose_action_performed": false, + "expected_host_ip_present": true, + "forbidden_action_seen": false, + "harbor_local_v2_http_status": 401, + "harbor_ready": true, + "host_reboot_performed": false, + "mode": "check", + "receipt_seen": true, + "repair_attempt_seen": false, + "service_restart_performed": false + } + }, + "rollups": { + "control_path_readiness_awoooi_host_unavailable": false, + "control_path_readiness_blocker_count": 2, + "control_path_readiness_node_high_load": false, + "control_path_readiness_non110_runner_unavailable": false, + "control_path_readiness_primary_blocker": "gitea_queue_cd_jobs_head_sha_mismatch", + "control_path_readiness_registry_v2_internal_ready": true, + "control_path_readiness_registry_v2_public_ready": true, + "control_path_readiness_runner_systemctl_timeout": false, + "control_path_readiness_status": "blocked_gitea_cd_jobs_readback_stale_or_mismatched", + "controlled_cd_lane_blocker_count": 8, + "controlled_cd_lane_readiness_receipt_seen": true, + "controlled_cd_lane_ready": false, + "controlled_cd_lane_safe_next_step": "restore_or_register_awoooi_cd_lane_drain_registration_without_printing_token_then_rerun_this_verifier", + "deploy_marker_blocker_count": 0, + "deploy_marker_latest_cd_run_status": "", + "deploy_marker_production_image_matches_expected": false, + "deploy_marker_readback_seen": false, + "deploy_marker_verified": false, + "gitea_queue_blocker_count": 4, + "gitea_queue_cd_jobs_head_sha_mismatch": true, + "gitea_queue_cd_jobs_payload_classifier": "cd_jobs_api_head_sha_mismatch_for_visible_cd_run", + "gitea_queue_cd_jobs_run_id_mismatch": false, + "gitea_queue_cd_jobs_stale_or_mismatched": true, + "gitea_queue_controlled_profile_no_matching_runner_label_count": 0, + "gitea_queue_current_cd_harbor_latest_registry_v2_status": "", + "gitea_queue_current_cd_harbor_repair_blocked_by_awoooi_host": false, + "gitea_queue_current_cd_harbor_repair_lane_classifier": "", + "gitea_queue_current_cd_harbor_repair_requires_110_controlled_lane": false, + "gitea_queue_current_cd_harbor_retrying_unavailable": false, + "gitea_queue_current_cd_inflight_classifier": "", + "gitea_queue_current_cd_no_matching_runner_label": "", + "gitea_queue_current_cd_waiting_behind_harbor_110_repair_running": false, + "gitea_queue_current_cd_waiting_for_runner_or_queue": false, + "gitea_queue_harbor_110_bounded_ssh_timeout_seen": false, + "gitea_queue_harbor_110_jobs_cross_workflow_mismatch": false, + "gitea_queue_harbor_110_jobs_payload_classifier": "unexpected_harbor_110_repair_job_names", + "gitea_queue_harbor_110_jobs_stale_or_mismatched": true, + "gitea_queue_harbor_110_no_matching_runner": false, + "gitea_queue_harbor_110_remote_control_channel_unavailable": true, + "gitea_queue_harbor_110_repair_failure_classifier": "harbor_110_remote_ssh_publickey_auth_stalled", + "gitea_queue_harbor_110_repair_run_status": "Failure", + "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale": false, + "gitea_queue_latest_cd_run_status": "Success", + "gitea_queue_normalized_classifier_field_count": 4, + "gitea_queue_normalized_classifier_field_ids": [ + "cd_run_jobs_payload_classifier", + "harbor_110_repair_jobs_payload_classifier", + "latest_visible_harbor_110_repair_no_matching_runner_label", + "harbor_110_repair_failure_classifier" + ], + "gitea_queue_readback_seen": true, + "local_console_blocked_phase_count": 1, + "local_console_completed_phase_count": 5, + "local_console_phase_count": 6, + "metadata_writeback_contract_ready": true, + "non110_runner_blocker_count": 0, + "non110_runner_readiness_receipt_seen": false, + "non110_runner_ready": false, + "non110_runner_ready_active_service_count": 0, + "non110_runner_ready_autostart_path_count": 0, + "non110_runner_ready_binary_count": 0, + "non110_runner_ready_config_count": 0, + "non110_runner_ready_registration_count": 0, + "non110_runner_ready_service_count": 0, + "non110_runner_safe_next_step": "", + "post_apply_verifier_ready": true, + "ssh_local_repair_account_metadata_ready": true, + "ssh_local_repair_control_channel_metadata_ready": true, + "ssh_local_repair_receipt_seen": true, + "ssh_local_repair_sshd_authorized_keys_file_default": true, + "ssh_local_repair_sshd_pubkeyauthentication": "yes", + "ssh_local_repair_target_user_account_locked": false, + "ssh_local_repair_target_user_shell_executable": true, + "ssh_publickey_diagnosis_ready": true, + "ssh_publickey_diagnosis_receipt_seen": true, + "ssh_publickey_node_exporter_ok": true, + "ssh_publickey_node_high_load_seen": false, + "ssh_publickey_node_load1_per_cpu": 0.41, + "ssh_publickey_node_load_classifier": "load_not_high", + "ssh_publickey_offer_timeout_effective_seen": false, + "ssh_publickey_offer_timeout_seen": false, + "ssh_publickey_port_tcp_open": true, + "ssh_publickey_runner_systemctl_show_timeout_effective_seen": false, + "ssh_publickey_runner_systemctl_show_timeout_seen": false, + "ssh_publickey_server_accepts_key_then_timeout_effective_seen": false, + "ssh_publickey_server_accepts_key_then_timeout_seen": true, + "ssh_publickey_wooo_command_path_classification": "command_path_ready", + "ssh_publickey_wooo_command_path_ready": true, + "ssh_publickey_wooo_publickey_classification": "server_accepts_key_then_timeout", + "watchdog_check_harbor_ready": true, + "watchdog_check_receipt_seen": true, + "watchdog_repair_harbor_ready": true, + "watchdog_repair_receipt_seen": true + }, + "safe_next_step": "fix_controlled_cd_lane_guardrail_blockers_then_rerun_readiness_verifier", + "schema_version": "harbor_registry_controlled_recovery_receipt_v1", + "scope": "harbor_registry_controlled_recovery_receipt", + "status": "controlled_cd_lane_readiness_receipt_blocked" + }, + "schema_version": "awoooi_harbor_110_control_path_recovery_readback_v1", + "source_control": { + "gitea_main_sha": "54f70ee011e5cabe475f56781544628a3930d55e", + "local_branch": "codex/mainline-runtime-continue-20260630", + "local_head_sha": "0ca72df655242d679b2564cc5dc1428b7da6588b" + } +} diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index e22b9dc71..e4db66e73 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.90 -> Last updated: 2026-07-01 Asia/Taipei +> Version: v1.91 +> Last updated: 2026-07-02 Asia/Taipei > Scope: 110 / 120 / 121 / 188 full-stack reboot recovery. 112 Kali is recorded as P3 optional and is not part of this recovery path. --- @@ -18,6 +18,8 @@ v1.79 active owner response template rule：同一輪 owner packet 產生後，p v1.80 / v1.81 credential escrow intake scorecard rule：同一輪 owner response preflight 後，必須用 `scripts/reboot-recovery/post-reboot-credential-escrow-intake-scorecard.py --summary-file "$ARTIFACT_DIR/summary.txt" --owner-packet-file --response-file --offsite-report-file --escrow-status-file ` 收斂 DR escrow gate。scorecard 只讀 sanitized artifacts；不得讀 secret value、不得寫 marker、不得送 owner request、不得開 runtime gate。placeholder readback 期望 `STATUS=blocked_waiting_non_secret_credential_escrow_evidence`、`EFFECTIVE_ESCROW_MISSING_COUNT=5`、`OWNER_RESPONSE_RECEIVED_COUNT=0`、`OWNER_RESPONSE_ACCEPTED_COUNT=0`、`RUNTIME_GATE_COUNT=0`、`CREDENTIAL_MARKER_WRITE_AUTHORIZED_COUNT=0`。若未來收到合格 redacted owner response 並由 preflight 回 `ready_for_independent_reviewer_acceptance`，scorecard 應轉為 `STATUS=ready_for_independent_reviewer_acceptance`；即使 marker 尚未寫入，也只能進 `independent_reviewer_acceptance_then_marker_dry_run`，不得直接寫 marker 或宣稱 `DR_COMPLETE`。 +2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 + 2026-07-01 23:00 latest live summary：core cold-start 已從 degraded 收斂為 GREEN，但仍不可宣稱 DR complete 或 MOMO 業績資料已最新。`full-stack-cold-start-check.sh --monitor-read-only --no-color --watch --interval 1 --max-attempts 1` artifact `/tmp/awoooi-cold-start-source-gate-20260701-225720.log` 回 `PASS=96 WARN=0 BLOCKED=0`、`Result: GREEN`。MOMO daily stale 在 `MOMO_DAILY_FRESHNESS 7|2026-06-24` 且 no-newer-source evidence 成立時不再算 core cold-start warning：cold-start 會輸出 `OK 188 momo daily sales source gate has no newer Drive candidate` 與 `INFO 188 momo daily sales data remains stale; product data freshness is pending source arrival`；這表示主機/服務恢復完成，但產品資料 freshness 仍留在 source-arrival gate，必須等正式 Drive source 到達後由原匯入 pipeline 更新，不得手動 DB 偽更新。110 live monitor 已同步，`/home/wooo/scripts/full-stack-cold-start-check.sh` hash `6115f73002b7e5b0fc46a031a2e7e9049d68abfcc8110f638e975218792c468e`；110 textfile 讀回 `awoooi_cold_start_monitor_up=1`、`pass=96`、`warn=0`、`blocked=0`、`last_exit_code=0`、`last_result{result="green"}=1`、`last_run_duration_seconds=26`。`verify-cold-start-monitor-deploy.sh` 回 `COLD_START_MONITOR_DEPLOY_PARITY_OK`，runtime state `monitor_up=1 warn=0 blocked=0 green=1 blocked_state=0`，ColdStart alerts `0`。`full-stack-recovery-scorecard.sh` 回 `CORE_COLD_START_GREEN=1`、`CORE_COLD_START_WARN_GATES=0`、`CORE_COLD_START_BLOCKED_GATES=0`、`CORE_COLD_START_FIRING_ALERTS=0`、`CORE_COLD_START_DEPLOY_PARITY=1`、`CORE_REGISTRY_READY=1`、`DR_OFFSITE_EVIDENCE_READBACK=1`、`ESCROW_MISSING_COUNT=5`、`NEXT_STEP=complete_credential_escrow_review`、`RECOVERY_STATE=CORE_READY_DR_OFFSITE_PENDING`。Allowed declaration：110 / 120 / 121 / 188 core cold-start service recovery GREEN，public routes / AWOOOI service / Gitea / Harbor registry / K3s / Stock public route / 188 backup-from-110 / 110 awoooi_db freshness 已恢復。Forbidden declaration：DR complete、credential escrow complete、MOMO data 最新、110 SSH 永久穩定、以假資料或手動 DB 寫入掩蓋 source freshness。 2026-07-01 21:32 previous live summary：cold-start 假 WARN 已收斂，hard blockers 維持 `0`，但仍不可宣稱 full green 或 10 分鐘全自動恢復完成。`full-stack-cold-start-check.sh --monitor-read-only --no-color --watch --interval 1 --max-attempts 1` final artifact `/tmp/awoooi-cold-start-final-20260701-212632.log` 回 `PASS=95 WARN=1 BLOCKED=0`；public routes / TLS 全部通過，StockPlatform 21:22 左右的 `502` 已確認是 web/admin/edge 替換 warmup，外部連續 5 次 `https://stock.wooo.work/` 回 `200`，final cold-start 亦回 `stock 200`。K3s `BAD_PODS=2` 也是 rollout 暫態，連續 6 次只讀觀察已無非 Running/Completed pod，final `BAD_PODS 0`。MOMO current-month `0|0|-|-|-|-` 不再列為 WARN：`momo-drive-token-source-recovery-preflight.sh` 會輸出 `MOMO_LATEST_IMPORT_CLEAN` 與 `MOMO_SOURCE_ABSENT_WITHOUT_NEWER_DRIVE`，cold-start 讀到 latest clean import 且 Drive 無更新 source candidate 時，判定 current-month sync not applicable。110 backup current health 也不再被舊 aggregate log 壓成 WARN：`BACKUP_HEALTH_110 total=13 stale=0 missing_cron=0 missing_script=0 failed_count=5 config_failed=0 integrity_total=2 integrity_stale=0` 代表 current component freshness / critical config / integrity OK；`failed_count=5` 保留為 INFO evidence，等下一次 full `backup-all` 自然覆蓋。live 110 monitor 已同步，hash `full-stack-cold-start-check.sh=d0711f75dfb1ee680442c9d6cf2191741f3b27605f347c9ef2a25a4fed6d40ac`、`momo-drive-token-source-recovery-preflight.sh=571d75e81c509683eb8a38fabbe81fc7822befe45206145f4fb4e865473f5254`；110 textfile 讀回 `awoooi_cold_start_monitor_up=1`、`pass=95`、`warn=1`、`blocked=0`、`last_exit_code=1`、`last_result{result="degraded"}=1`。`verify-cold-start-monitor-deploy.sh` 回 `COLD_START_MONITOR_DEPLOY_PARITY_OK`；`full-stack-recovery-scorecard.sh` 回 `CORE_COLD_START_WARN_GATES=1`、`CORE_COLD_START_BLOCKED_GATES=0`、`CORE_COLD_START_DEPLOY_PARITY=1`、`CORE_REGISTRY_READY=1`、`DR_OFFSITE_EVIDENCE_READBACK=1`、`ESCROW_MISSING_COUNT=5`、`NEXT_STEP=complete_credential_escrow_review`。Allowed declaration：public routes / AWOOOI service / Gitea / Harbor registry / 188 backup-from-110 / 110 awoooi_db freshness / K3s rollout / Stock public route 已恢復，cold-start hard blockers `0`。Forbidden declaration：full green、10 分鐘全自動恢復完成、DR complete、credential escrow complete、MOMO data 最新、110 SSH 永久穩定。唯一 cold-start WARN 是 MOMO daily data freshness：`MOMO_DAILY_FRESHNESS 7|2026-06-24`，且 Drive intake / failed folder 無新候選；必須走 source-arrival / formal import gate，不可用假資料或手動 DB 寫入掩蓋。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 3671ea52c..a0aaf35ed 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.0 -> **最後更新**: 2026-04-05 下午 (台北時間) -> **更新者**: Claude Code (首席架構師) -> **觸發事件**: Harbor Exited(128) Race Condition 根治 + harbor-watchdog 常駐自愈 +> **版本**: v5.1 +> **最後更新**: 2026-07-02 (台北時間) +> **更新者**: Codex +> **觸發事件**: 110 control-path / Harbor recovery receipt 與 Gitea stale queue blocker 收斂 --- @@ -353,6 +353,22 @@ systemctl status postgresql@14-main --no-pager | 8 | Gitea Runner | `docker logs gitea-runner \| grep SUCCESS` | .runner 配置過期 | | 9 | Sentry | `curl -o /dev/null -w '%{http_code}' http://localhost:9000/` | PostgreSQL WAL/Redis RDB 損壞 (見下方) | +**110 control-path / Harbor receipt 判讀 (2026-07-02)**: + +重啟後若公開 Gitea queue 仍顯示舊的 Harbor repair failure、`harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable 或 jobs stale，不得直接把它當成 current blocker。必須先用同一輪 read-only receipt 合併以下證據： + +```bash +scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh +ssh wooo@192.168.0.110 'sudo -n /usr/local/bin/recover-110-control-path-and-harbor-local.sh --check' +python3 ops/runner/read-public-gitea-actions-queue.py --json +curl -k -s -o /dev/null -w '%{http_code}' https://registry.wooo.work/v2/ +curl -s -o /dev/null -w '%{http_code}' http://192.168.0.110:5000/v2/ +``` + +若 `SSH_COMMAND_PATH ... classification=command_path_ready` 成立，且 public/internal registry `/v2/` 回 `401`，則舊 Harbor repair SSH failure 只能列為 historical queue metadata；active blocker 應轉向實際的 controlled CD lane readiness、CD jobs SHA/readback mismatch 或 deploy marker。2026-07-02 的機器可讀證據是 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json`：Gitea latest visible CD `#4335` 為 `Success`，但 active blockers 仍包含 110 controlled CD lane config / binary / registration / service guardrail 與 Gitea CD jobs head-SHA / stale mismatch。 + +combined local-console output 只能依 marker 判讀：`AWOOOI_110_CONTROLLED_CD_LANE_READY` 只屬於 110 controlled lane；沒有 `AWOOOI_NON110_RUNNER_READY` 時，不得從同一份 `BLOCKER` 行推導 non110 runner blocker。 + **Harbor Exited 128 修復**: ```bash # 等 harbor-log healthy