diff --git a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py index bd1407393..7eb4014dc 100644 --- a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py @@ -151,6 +151,15 @@ def validate_harbor_registry_controlled_recovery_receipt( "ssh_publickey_runner_systemctl_show_timeout_seen": ( ssh_diagnosis["runner_systemctl_show_timeout_seen"] ), + "ssh_publickey_node_high_load_seen": ssh_diagnosis[ + "node_high_load_seen" + ], + "ssh_publickey_node_load_classifier": ssh_diagnosis[ + "node_load_classifier" + ], + "ssh_publickey_node_load1_per_cpu": ssh_diagnosis[ + "node_load1_per_cpu" + ], "ssh_publickey_node_exporter_ok": ssh_diagnosis["node_exporter_ok"], "ssh_publickey_port_tcp_open": ssh_diagnosis["ssh_port_tcp_open"], "ssh_local_repair_receipt_seen": ssh_local["receipt_seen"], @@ -375,12 +384,14 @@ def _parse_ssh_publickey_diagnosis_output(output: str) -> dict[str, Any]: ) ssh_port_tcp_open = str(fields.get("SSH_PORT") or "") == "tcp_open" node_exporter_ok = str(fields.get("NODE_EXPORTER") or "") == "ok" + node_high_load_seen = str(fields.get("NODE_LOAD_CLASSIFIER") or "") == "high_load" return { "receipt_seen": marker_seen, "target": str(fields.get("TARGET") or ""), "node_exporter_ok": node_exporter_ok, "node_load_classifier": str(fields.get("NODE_LOAD_CLASSIFIER") or ""), "node_load1_per_cpu": _float_or_none(fields.get("NODE_LOAD1_PER_CPU")), + "node_high_load_seen": node_high_load_seen, "ssh_port_tcp_open": ssh_port_tcp_open, "ssh_banner_seen": "SSH_BANNER=SSH-" in output, "auth_attempt_count": len(auth_attempts), @@ -848,6 +859,8 @@ def _active_blockers( blockers.append("ssh_publickey_server_accepts_key_then_timeout_on_wooo") if ssh_diagnosis["runner_systemctl_show_timeout_seen"]: blockers.append("runner_systemctl_show_timeout_on_110") + if ssh_diagnosis["node_high_load_seen"]: + blockers.append("ssh_publickey_node_high_load_on_110") if ssh_local["receipt_seen"] and not ssh_local["control_channel_metadata_ready"]: blockers.append("ssh_local_repair_receipt_metadata_not_ready") if not watchdog_check["receipt_seen"]: diff --git a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py index 676ef645f..903421e17 100644 --- a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py @@ -130,6 +130,8 @@ def test_harbor_recovery_receipt_accepts_ssh_publickey_diagnosis() -> None: ) assert diagnosis["server_accepts_key_then_timeout_seen"] is True assert diagnosis["runner_systemctl_show_timeout_seen"] is True + assert diagnosis["node_load_classifier"] == "load_not_high" + assert diagnosis["node_high_load_seen"] is False assert diagnosis["systemd_unit_signals"][0]["classifier"] == ( "systemctl_show_timeout" ) @@ -150,6 +152,12 @@ def test_harbor_recovery_receipt_accepts_ssh_publickey_diagnosis() -> None: payload["rollups"]["ssh_publickey_runner_systemctl_show_timeout_seen"] is True ) + assert payload["rollups"]["ssh_publickey_node_high_load_seen"] is False + assert ( + payload["rollups"]["ssh_publickey_node_load_classifier"] + == "load_not_high" + ) + assert payload["rollups"]["ssh_publickey_node_load1_per_cpu"] == 0.93 assert payload["input_redaction"]["ssh_publickey_diagnosis_output"][ "line_count" ] > 0 @@ -159,6 +167,12 @@ def test_harbor_recovery_receipt_classifies_publickey_offer_timeout() -> None: diagnosis_output = _ssh_publickey_diagnosis_output().replace( "rc=124 classification=server_accepts_key_then_timeout", "rc=255 classification=publickey_offer_timeout", + ).replace( + "NODE_LOAD1_PER_CPU=0.93", + "NODE_LOAD1_PER_CPU=1.87", + ).replace( + "NODE_LOAD_CLASSIFIER=load_not_high", + "NODE_LOAD_CLASSIFIER=high_load", ) payload = validate_harbor_registry_controlled_recovery_receipt( @@ -168,9 +182,13 @@ def test_harbor_recovery_receipt_classifies_publickey_offer_timeout() -> None: ) assert "ssh_publickey_offer_timeout_on_wooo" in payload["active_blockers"] + assert "ssh_publickey_node_high_load_on_110" in payload["active_blockers"] diagnosis = payload["readback"]["ssh_publickey_diagnosis"] assert diagnosis["wooo_publickey_classification"] == "publickey_offer_timeout" assert diagnosis["publickey_offer_timeout_seen"] is True + assert diagnosis["node_high_load_seen"] is True + assert diagnosis["node_load1_per_cpu"] == 1.87 + assert payload["rollups"]["ssh_publickey_node_high_load_seen"] is True phases = { phase["phase_id"]: phase for phase in payload["local_console_phase_readback"]["phases"] diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index bc12b62e0..4cb4ff26c 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,19 @@ +## 2026-07-01 — 110 SSH high-load receipt blocker + +**照主線修正的問題**： +- 最新 Gitea main fast-forward 到 `77f9bb041 fix(recovery): classify harbor repair cross-workflow jobs`；本輪 live queue 仍回 `status=blocked_harbor_110_repair_no_matching_runner`，Harbor repair `#4173 Waiting`，no-matching label `awoooi-host`，jobs API 仍 stale / mismatched。 +- Live route probe 仍回 `registry.wooo.work/v2=502`、`harbor.wooo.work/api/v2.0/health=502`、`192.168.0.110:5000/v2=502`；110 shell SSH bounded publickey probe 仍是 `Connection to 192.168.0.110 port 22 timed out`。 +- `diagnose-110-ssh-publickey-auth.sh` read-only diagnosis 回 `NODE_EXPORTER=ok`、`SSH_PORT=tcp_open`、OpenSSH banner 可見、`SSH_AUTH user=wooo mode=publickey classification=publickey_offer_timeout`，同時 `NODE_LOAD1_PER_CPU=1.87` / `NODE_LOAD_CLASSIFIER=high_load`，多個 `actions.runner.*` systemd unit 回 `systemctl_show_timeout`。 +- `harbor_registry_controlled_recovery_receipt` 已能解析 `NODE_LOAD_CLASSIFIER`，但原本未把 high load 納入 active blocker；本輪新增 `ssh_publickey_node_high_load_on_110` blocker 與 rollups：`ssh_publickey_node_high_load_seen`、`ssh_publickey_node_load_classifier`、`ssh_publickey_node_load1_per_cpu`。 +- 用 live diagnosis + queue metadata 重跑 receipt validator 回 `status=ssh_publickey_diagnosis_receipt_waiting_110_local_repair_or_harbor_watchdog_check`、`safe_next_step=run_110_local_ssh_metadata_check_then_harbor_watchdog_check_mode`；5 個 local-console phase 中只有 `diagnose_ssh_publickey` ready，其餘仍等待 SSH metadata repair、Harbor watchdog check/repair 與 registry verifier。 + +**驗證**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost:5432/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py -q` 通過（9 passed）。 +- `ruff check`、`py_compile`、`git diff --check` 通過。 +- 前置同步驗證最新 main focused chain：`pytest apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py ops/runner/test_read_public_gitea_actions_queue.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py -q` 通過（58 passed）；`guard-gitea-runner-pressure.py --root .` 與 Gitea secrets guard 通過。 + +**邊界**：只改 receipt validator / tests / LOGBOOK；只讀 public Gitea queue、HTTP probes、node_exporter metrics 與 bounded SSH classification；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未 SSH 寫入、未重啟主機、未 restart Docker / Nginx / K3s / DB / firewall。 + ## 2026-06-30 — 23:30 110 startup guarded auto-starts controlled drain lane **照主線修正的問題**：