diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 65267d417..af103ea2f 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -342,6 +342,8 @@ jobs: ;; apps/api/src/services/stockplatform_public_api_controlled_recovery_preflight.py) ;; + apps/api/src/services/harbor_registry_controlled_recovery_preflight.py) + ;; apps/api/src/services/iwooos_security_operating_system.py) ;; apps/api/Dockerfile) @@ -448,6 +450,8 @@ jobs: ;; apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py) ;; + apps/api/tests/test_harbor_registry_controlled_recovery_preflight.py) + ;; apps/api/tests/test_iwooos_security_operating_system.py) ;; apps/api/tests/test_iwooos_wazuh_prod_manifest.py) @@ -502,6 +506,8 @@ jobs: ;; scripts/reboot-recovery/full-stack-recovery-scorecard.sh) ;; + scripts/reboot-recovery/harbor-watchdog.sh) + ;; scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh) ;; scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh) @@ -534,6 +540,8 @@ jobs: ;; scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py) ;; + scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py) + ;; scripts/security/gitea-private-inventory-p0-scorecard.py) ;; scripts/security/gitea-authenticated-inventory-payload-validator.py) @@ -656,6 +664,7 @@ jobs: src/services/reboot_auto_recovery_drill_preflight.py \ src/services/stockplatform_public_api_runtime_readback.py \ src/services/stockplatform_public_api_controlled_recovery_preflight.py \ + src/services/harbor_registry_controlled_recovery_preflight.py \ src/services/iwooos_security_operating_system.py \ src/services/awoooi_gitea_onboarding_warning_step_dashboard.py \ src/services/awoooi_gitea_onboarding_warning_step_owner_package.py \ @@ -689,6 +698,7 @@ jobs: ../../scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh \ ../../scripts/reboot-recovery/full-stack-cold-start-check.sh \ ../../scripts/reboot-recovery/full-stack-recovery-scorecard.sh \ + ../../scripts/reboot-recovery/harbor-watchdog.sh \ ../../scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh \ ../../scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh \ ../../scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh \ @@ -720,6 +730,7 @@ jobs: tests/test_reboot_auto_recovery_slo_scorecard_api.py \ tests/test_stockplatform_public_api_runtime_readback.py \ tests/test_stockplatform_public_api_controlled_recovery_preflight.py \ + tests/test_harbor_registry_controlled_recovery_preflight.py \ tests/test_iwooos_security_operating_system.py \ tests/test_awoooi_production_deploy_readback_blocker.py \ tests/test_awoooi_priority_work_order_readback_api.py \ @@ -737,6 +748,7 @@ jobs: ../../scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py \ ../../scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py \ ../../scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py \ + ../../scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py \ ../../scripts/security/tests/test_gitea_private_inventory_p0_scorecard.py \ ../../scripts/security/tests/test_gitea_authenticated_inventory_payload_validator.py \ -v --tb=short -x -p no:cacheprovider \ diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index 50bd58094..851985344 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -336,6 +336,7 @@ from src.services.awoooi_gitea_onboarding_warning_step_template_copy_receipt imp load_latest_awoooi_gitea_onboarding_warning_step_template_copy_receipt, ) from src.services.awoooi_priority_work_order_readback import ( + apply_harbor_registry_controlled_recovery_preflight, apply_stockplatform_public_api_controlled_recovery_preflight, apply_stockplatform_public_api_runtime_readback, load_latest_awoooi_priority_work_order_readback, @@ -402,6 +403,9 @@ from src.services.github_target_private_backup_evidence_gate import ( preflight_github_target_owner_response_submission, validate_github_target_safe_credential_evidence_refs, ) +from src.services.harbor_registry_controlled_recovery_preflight import ( + load_latest_harbor_registry_controlled_recovery_preflight, +) from src.services.host_runaway_aiops_loop_readiness import ( load_latest_host_runaway_aiops_loop_readiness, ) @@ -1100,6 +1104,13 @@ async def get_awoooi_priority_work_order_readback() -> dict[str, Any]: payload, stockplatform_recovery, ) + harbor_recovery = await asyncio.to_thread( + load_latest_harbor_registry_controlled_recovery_preflight + ) + apply_harbor_registry_controlled_recovery_preflight( + payload, + harbor_recovery, + ) return redact_public_lan_topology(payload) except FileNotFoundError as exc: raise HTTPException( @@ -1172,6 +1183,36 @@ async def get_stockplatform_public_api_controlled_recovery_preflight() -> dict[s ) from exc +@router.get( + "/harbor-registry-controlled-recovery-preflight", + response_model=dict[str, Any], + summary="取得 Harbor registry controlled recovery preflight", + description=( + "讀取 Harbor / registry public 與 110 internal /v2/ readback,並產生 " + "AI controlled recovery preflight:target selector、watchdog executor " + "contract、bounded action candidates、post-verifier 與 " + "KM/RAG/MCP/PlayBook writeback contract。此端點不 SSH、不 Docker、" + "不 restart、不讀 secret、不觸發 workflow、不寫 runtime。" + ), +) +async def get_harbor_registry_controlled_recovery_preflight() -> dict[str, Any]: + """回傳 Harbor registry controlled recovery preflight。""" + try: + payload = await asyncio.to_thread( + load_latest_harbor_registry_controlled_recovery_preflight + ) + return redact_public_lan_topology(payload) + except (json.JSONDecodeError, ValueError) as exc: + logger.error( + "harbor_registry_controlled_recovery_preflight_invalid", + error=str(exc), + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Harbor registry controlled recovery preflight 無效", + ) from exc + + @router.get( "/product-awoooi-manifest-standard", response_model=dict[str, Any], diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index 66cb9afdf..cb0fb6d58 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -351,6 +351,209 @@ def apply_stockplatform_public_api_controlled_recovery_preflight( ) +def apply_harbor_registry_controlled_recovery_preflight( + payload: dict[str, Any], + recovery_preflight: dict[str, Any], +) -> None: + """Overlay the packaged Harbor registry recovery preflight.""" + status = str(recovery_preflight.get("status") or "unknown") + if status == "harbor_registry_ready": + return + + state = _dict(payload.setdefault("mainline_execution_state", {})) + rollups = _dict(recovery_preflight.get("rollups")) + controlled_policy = _dict(recovery_preflight.get("controlled_apply_policy")) + candidate_packaged = bool( + rollups.get("controlled_recovery_candidate_packaged") is True + ) + + state["harbor_registry_controlled_recovery_preflight_status"] = status + state["harbor_registry_controlled_recovery_active_blockers"] = _strings( + recovery_preflight.get("active_blockers") + ) + state["harbor_registry_controlled_recovery_candidate_packaged"] = ( + candidate_packaged + ) + state["active_p0_state"] = "blocked_harbor_registry_controlled_recovery_preflight" + state["next_executable_mainline_workplan_id"] = ( + "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT" + ) + state["next_executable_mainline_state"] = _harbor_registry_next_state( + status=status, + candidate_packaged=candidate_packaged, + ) + state["active_p0_live_active_blockers"] = _unique_strings( + _strings(state.get("active_p0_live_active_blockers")) + + ["harbor_registry_controlled_recovery_preflight"] + + _strings(recovery_preflight.get("active_blockers")) + ) + + for item in _list(payload.get("in_progress_or_blocked_in_priority_order")): + workplan = _dict(item) + if workplan.get("workplan_id") != "P0-006": + continue + + evidence = _dict(workplan.setdefault("evidence", {})) + evidence["harbor_registry_controlled_recovery_preflight_status"] = status + evidence["harbor_registry_controlled_recovery_candidate_packaged"] = ( + candidate_packaged + ) + evidence["harbor_registry_public_v2_ready"] = bool( + rollups.get("public_registry_v2_ready") is True + ) + evidence["harbor_registry_internal_v2_ready"] = bool( + rollups.get("internal_110_registry_v2_ready") is True + ) + evidence["harbor_registry_upstream_502_shape"] = bool( + rollups.get("harbor_registry_upstream_502_shape") is True + ) + evidence["harbor_registry_public_gateway_route_drift_shape"] = bool( + rollups.get("public_gateway_route_drift_shape") is True + ) + evidence["harbor_registry_watchdog_executor_ready"] = bool( + rollups.get("watchdog_executor_ready") is True + ) + evidence["harbor_registry_controlled_work_item_count"] = _int( + rollups.get("controlled_work_item_count") + ) + evidence["harbor_registry_post_apply_verifier_count"] = _int( + rollups.get("post_apply_verifier_count") + ) + evidence["harbor_registry_learning_writeback_contract_count"] = _int( + rollups.get("learning_writeback_contract_count") + ) + evidence["harbor_registry_current_apply_allowed"] = bool( + controlled_policy.get("current_apply_allowed") is True + ) + evidence["harbor_registry_runtime_write_gate"] = str( + controlled_policy.get("runtime_write_gate") or "" + ) + + workplan["status"] = "blocked_harbor_registry_controlled_recovery_preflight" + workplan["safe_next_step"] = str( + recovery_preflight.get("safe_next_step") or "" + ) + workplan["reason"] = _harbor_registry_reason( + status=status, + candidate_packaged=candidate_packaged, + ) + professional_fix = _dict(workplan.setdefault("professional_fix", {})) + professional_fix["action"] = _harbor_registry_action(status=status) + professional_fix["owner"] = ( + "Harbor registry controlled recovery preflight plus P0-006 deploy " + "closure lane" + ) + + payload["status"] = "p0_006_blocked_harbor_registry_controlled_recovery_preflight" + payload["next_execution_order"] = [ + _harbor_registry_first_order(status=status), + ( + "P0-006-CD-DEPLOY-MARKER-READBACK: after registry /v2/ is 200/401, " + "let the next Gitea CD run build/push/deploy and then verify " + "delivery-closure-workbench, priority work order, and production " + "marker advance." + ), + ( + "P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT: " + "resume StockPlatform API/data dependency recovery only after the " + "AWOOOI API deploy marker can advance through Harbor." + ), + ( + "P0-006: keep reboot SLO timer live, but do not claim final SLO " + "closure while deploy marker and StockPlatform readback are blocked." + ), + ] + _refresh_rollups_after_stockplatform_overlay(payload, state) + summary = _dict(payload.setdefault("summary", {})) + summary["harbor_registry_controlled_recovery_preflight_status"] = status + summary["harbor_registry_controlled_recovery_candidate_packaged"] = ( + candidate_packaged + ) + summary["harbor_registry_public_v2_ready"] = bool( + rollups.get("public_registry_v2_ready") is True + ) + summary["harbor_registry_internal_v2_ready"] = bool( + rollups.get("internal_110_registry_v2_ready") is True + ) + summary["harbor_registry_watchdog_executor_ready"] = bool( + rollups.get("watchdog_executor_ready") is True + ) + + +def _harbor_registry_next_state(*, status: str, candidate_packaged: bool) -> str: + if candidate_packaged and "upstream_502" in status: + return ( + "controlled_recovery_packaged_waiting_harbor_watchdog_check_mode_" + "and_repair_once_receipt" + ) + if candidate_packaged and "public_gateway_route_drift" in status: + return ( + "controlled_recovery_packaged_waiting_188_registry_nginx_rendered_" + "diff_and_nginx_t" + ) + return "controlled_recovery_waiting_registry_v2_evidence_and_executor_contract" + + +def _harbor_registry_reason(*, status: str, candidate_packaged: bool) -> str: + if candidate_packaged and "upstream_502" in status: + return ( + "Current Gitea CD cannot deploy AWOOOI because Harbor registry /v2/ " + "is returning 502 while the Harbor UI is reachable. The active " + "mainline item is now a controlled Harbor watchdog check-mode / " + "repair-once package with post-apply registry verifier and " + "KM/RAG/MCP/PlayBook writeback. This is not a manual end state." + ) + if candidate_packaged and "public_gateway_route_drift" in status: + return ( + "Current Gitea CD cannot deploy AWOOOI because the public registry " + "route is unhealthy while the internal registry /v2/ readback is " + "healthy. The active mainline item is a controlled 188 Nginx " + "rendered diff / nginx -t / route smoke package, not a manual end " + "state." + ) + return ( + "Harbor registry recovery is active and waiting for registry /v2/ " + "shape evidence plus executor contract before any bounded apply; this " + "is not a manual end state." + ) + + +def _harbor_registry_action(*, status: str) -> str: + if "public_gateway_route_drift" in status: + return ( + "Prepare the 188 registry public gateway controlled route diff, run " + "rendered diff and nginx -t, keep rollback backup and route smoke " + "receipts, then verify registry /v2/ and rerun CD marker readback. " + "Do not switch registry provider, reload Nginx without check-mode, " + "restart Docker daemon, reboot hosts, trigger workflows, or read " + "secrets from this lane." + ) + return ( + "Run Harbor registry controlled recovery in order: verify 110 control " + "channel, execute harbor-watchdog.sh --check, run " + "--repair-once only if check-mode proves /v2/ unhealthy, rerun public " + "and internal registry /v2/ verifier, then allow CD marker readback and " + "write metadata-only KM/RAG/MCP/PlayBook receipts. Do not restart " + "Docker daemon, reboot hosts, drain nodes, reload Nginx, switch " + "registry provider, trigger workflows, or read secrets from this lane." + ) + + +def _harbor_registry_first_order(*, status: str) -> str: + if "public_gateway_route_drift" in status: + return ( + "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: registry " + "public route drift is blocking AWOOOI deploy marker; run 188 " + "rendered diff / nginx -t / route smoke controlled package first." + ) + return ( + "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: Harbor registry " + "/v2/ 502 is blocking AWOOOI deploy marker; run watchdog check-mode, " + "bounded repair-once, registry /v2/ verifier, and metadata writeback " + "before retrying CD." + ) + + def _stockplatform_recovery_next_state( *, candidate_packaged: bool, diff --git a/apps/api/src/services/harbor_registry_controlled_recovery_preflight.py b/apps/api/src/services/harbor_registry_controlled_recovery_preflight.py new file mode 100644 index 000000000..4b0c4b64d --- /dev/null +++ b/apps/api/src/services/harbor_registry_controlled_recovery_preflight.py @@ -0,0 +1,641 @@ +"""Harbor registry controlled recovery preflight. + +This readback packages the current CD-blocking Harbor /v2/ failure into a +controlled recovery candidate. It probes only public/internal HTTP routes and +repo source files; it does not SSH, read secrets, run Docker, restart services, +trigger workflows, or mutate runtime state. +""" + +from __future__ import annotations + +import re +import urllib.error +import urllib.request +from collections.abc import Callable +from pathlib import Path +from typing import Any + +_SCHEMA_VERSION = "harbor_registry_controlled_recovery_preflight_v1" +_DEFAULT_ROUTE_SOURCE = ( + Path(__file__).resolve().parents[4] + / "infra" + / "ansible" + / "roles" + / "nginx" + / "templates" + / "188-internal-tools-https.conf.j2" +) +_DEFAULT_WATCHDOG_SOURCE = ( + Path(__file__).resolve().parents[4] + / "scripts" + / "reboot-recovery" + / "harbor-watchdog.sh" +) +_DEFAULT_TIMEOUT_SECONDS = 4.0 + +Probe = Callable[[str, float], dict[str, Any]] + + +def load_latest_harbor_registry_controlled_recovery_preflight( + *, + timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS, + route_source_path: Path | None = None, + watchdog_source_path: Path | None = None, + probe: Probe | None = None, +) -> dict[str, Any]: + """Build the Harbor registry controlled recovery preflight.""" + http_probe = probe or _http_probe + probes = { + "public_registry_v2": _probe_endpoint( + http_probe, + "https://registry.wooo.work/v2/", + timeout_seconds, + ), + "public_harbor_ui": _probe_endpoint( + http_probe, + "https://harbor.wooo.work/", + timeout_seconds, + ), + "public_harbor_api_health": _probe_endpoint( + http_probe, + "https://harbor.wooo.work/api/v2.0/health", + timeout_seconds, + ), + "internal_110_registry_v2": _probe_endpoint( + http_probe, + "http://192.168.0.110:5000/v2/", + timeout_seconds, + ), + "internal_110_harbor_ui": _probe_endpoint( + http_probe, + "http://192.168.0.110:5000/", + timeout_seconds, + ), + } + route_contract = _load_route_contract(route_source_path or _DEFAULT_ROUTE_SOURCE) + watchdog_contract = _load_watchdog_contract( + watchdog_source_path or _DEFAULT_WATCHDOG_SOURCE + ) + return _build_payload( + timeout_seconds=timeout_seconds, + probes=probes, + route_contract=route_contract, + watchdog_contract=watchdog_contract, + ) + + +def _build_payload( + *, + timeout_seconds: float, + probes: dict[str, dict[str, Any]], + route_contract: dict[str, Any], + watchdog_contract: dict[str, Any], +) -> dict[str, Any]: + public_v2 = _dict(probes.get("public_registry_v2")) + public_ui = _dict(probes.get("public_harbor_ui")) + public_health = _dict(probes.get("public_harbor_api_health")) + internal_v2 = _dict(probes.get("internal_110_registry_v2")) + internal_ui = _dict(probes.get("internal_110_harbor_ui")) + + public_v2_ready = _registry_ready(public_v2.get("http_status")) + internal_v2_ready = _registry_ready(internal_v2.get("http_status")) + public_ui_ready = public_ui.get("http_status") == 200 + internal_ui_ready = internal_ui.get("http_status") == 200 + route_target_ready = route_contract.get("route_target_ready") is True + watchdog_executor_ready = watchdog_contract.get("executor_ready") is True + upstream_502_shape = ( + public_v2.get("http_status") == 502 + and internal_v2.get("http_status") == 502 + and (public_ui_ready or internal_ui_ready) + ) + gateway_drift_shape = ( + public_v2.get("http_status") == 502 + and internal_v2_ready + and route_target_ready + ) + registry_ready = public_v2_ready and internal_v2_ready + candidate_packaged = ( + not registry_ready + and route_target_ready + and watchdog_executor_ready + and (upstream_502_shape or gateway_drift_shape) + ) + status = _status( + registry_ready=registry_ready, + candidate_packaged=candidate_packaged, + upstream_502_shape=upstream_502_shape, + gateway_drift_shape=gateway_drift_shape, + ) + active_blockers = _active_blockers( + registry_ready=registry_ready, + route_target_ready=route_target_ready, + watchdog_executor_ready=watchdog_executor_ready, + upstream_502_shape=upstream_502_shape, + gateway_drift_shape=gateway_drift_shape, + public_v2_status=public_v2.get("http_status"), + internal_v2_status=internal_v2.get("http_status"), + ) + + return { + "schema_version": _SCHEMA_VERSION, + "priority": "P0-006", + "scope": "harbor_registry_controlled_recovery_preflight", + "status": status, + "safe_next_step": _safe_next_step( + registry_ready=registry_ready, + upstream_502_shape=upstream_502_shape, + gateway_drift_shape=gateway_drift_shape, + candidate_packaged=candidate_packaged, + ), + "active_blockers": active_blockers, + "active_blocker_count": len(active_blockers), + "timeout_seconds": timeout_seconds, + "target_selector": { + "component_id": "harbor_registry", + "public_registry_route": "https://registry.wooo.work/v2/", + "public_harbor_ui_route": "https://harbor.wooo.work/", + "internal_registry_route": "http://192.168.0.110:5000/v2/", + "internal_harbor_ui_route": "http://192.168.0.110:5000/", + "source_route_file": str(route_contract.get("route_source_file") or ""), + "route_targets": _strings(route_contract.get("route_targets")), + "watchdog_source_file": str( + watchdog_contract.get("watchdog_source_file") or "" + ), + "watchdog_check_command": "harbor-watchdog.sh --check", + "watchdog_repair_command": "harbor-watchdog.sh --repair-once", + }, + "source_of_truth_diff": { + "status": _source_diff_status( + registry_ready=registry_ready, + route_target_ready=route_target_ready, + upstream_502_shape=upstream_502_shape, + gateway_drift_shape=gateway_drift_shape, + ), + "public_registry_v2_http_status": public_v2.get("http_status"), + "public_harbor_ui_http_status": public_ui.get("http_status"), + "public_harbor_api_health_http_status": public_health.get("http_status"), + "internal_110_registry_v2_http_status": internal_v2.get("http_status"), + "internal_110_harbor_ui_http_status": internal_ui.get("http_status"), + "route_source_file": str(route_contract.get("route_source_file") or ""), + "route_targets": _strings(route_contract.get("route_targets")), + "watchdog_executor_ready": watchdog_executor_ready, + }, + "controlled_apply_policy": { + "risk_level": "high", + "break_glass_required": False, + "runtime_write_gate": "controlled_after_preflight", + "controlled_recovery_candidate_packaged": candidate_packaged, + "current_apply_allowed": False, + "current_apply_blocker": _current_apply_blocker( + registry_ready=registry_ready, + candidate_packaged=candidate_packaged, + upstream_502_shape=upstream_502_shape, + gateway_drift_shape=gateway_drift_shape, + ), + "manual_end_state": False, + "allowed_scope": [ + "public_registry_v2_reprobe", + "read_only_110_local_v2_status", + "harbor_watchdog_check_mode", + "harbor_watchdog_repair_once_after_status_only_failure", + "post_apply_public_registry_v2_verifier", + "km_rag_mcp_playbook_metadata_writeback", + ], + "forbidden_scope": [ + "docker_daemon_restart", + "host_reboot", + "node_drain", + "nginx_reload_or_restart", + "database_restore_or_prune", + "secret_value_read", + "workflow_dispatch", + "force_push_or_repo_ref_mutation", + "switch_registry_provider_or_paid_route", + ], + }, + "controlled_work_items": _controlled_work_items( + candidate_packaged=candidate_packaged, + upstream_502_shape=upstream_502_shape, + gateway_drift_shape=gateway_drift_shape, + ), + "controlled_action_candidates": _controlled_action_candidates( + candidate_packaged=candidate_packaged, + upstream_502_shape=upstream_502_shape, + gateway_drift_shape=gateway_drift_shape, + ), + "post_apply_verifiers": _post_apply_verifiers(), + "learning_writeback_contracts": _learning_writeback_contracts(), + "rollups": { + "registry_ready": registry_ready, + "public_registry_v2_ready": public_v2_ready, + "internal_110_registry_v2_ready": internal_v2_ready, + "public_harbor_ui_ready": public_ui_ready, + "internal_110_harbor_ui_ready": internal_ui_ready, + "route_target_ready": route_target_ready, + "watchdog_executor_ready": watchdog_executor_ready, + "harbor_registry_upstream_502_shape": upstream_502_shape, + "public_gateway_route_drift_shape": gateway_drift_shape, + "controlled_recovery_candidate_packaged": candidate_packaged, + "controlled_work_item_count": len( + _controlled_work_items( + candidate_packaged=candidate_packaged, + upstream_502_shape=upstream_502_shape, + gateway_drift_shape=gateway_drift_shape, + ) + ), + "post_apply_verifier_count": len(_post_apply_verifiers()), + "learning_writeback_contract_count": len(_learning_writeback_contracts()), + }, + "probes": probes, + "operation_boundaries": { + "read_only_public_https_probe": True, + "read_only_internal_http_probe": True, + "source_route_file_read": True, + "watchdog_source_file_read": True, + "ssh_used": False, + "docker_command_performed": False, + "docker_restart_performed": False, + "docker_daemon_restart_performed": False, + "host_reboot_performed": False, + "node_drain_performed": False, + "nginx_reload_or_restart_performed": False, + "database_write_or_restore_performed": False, + "secret_value_collection_allowed": False, + "github_api_used": False, + "workflow_trigger_performed": False, + "runtime_write_allowed_by_this_readback": False, + }, + } + + +def _load_route_contract(path: Path) -> dict[str, Any]: + payload: dict[str, Any] = { + "route_source_file": str(path), + "route_targets": [], + "route_target_ready": False, + "route_source_present": path.exists(), + } + if not path.exists(): + return payload + text = path.read_text(encoding="utf-8") + section = _registry_section(text) + targets = re.findall(r"proxy_pass\s+(http://[^;]+);", section) + unique_targets = _unique_strings(targets) + payload["route_targets"] = unique_targets + payload["route_target_ready"] = ( + "server_name registry.wooo.work;" in section + and "server_name harbor.wooo.work;" in text + and bool(unique_targets) + ) + return payload + + +def _registry_section(text: str) -> str: + marker = "server_name registry.wooo.work;" + start = text.find(marker) + if start == -1: + return text + block_start = text.rfind("server {", 0, start) + block_end = text.find("\n}", start) + if block_start == -1 or block_end == -1: + return text[start:] + return text[block_start : block_end + 2] + + +def _load_watchdog_contract(path: Path) -> dict[str, Any]: + payload: dict[str, Any] = { + "watchdog_source_file": str(path), + "watchdog_source_present": path.exists(), + "executor_ready": False, + } + if not path.exists(): + return payload + text = path.read_text(encoding="utf-8") + payload["has_check_mode"] = "--check" in text + payload["has_repair_once"] = "--repair-once" in text + payload["has_v2_health_check"] = "http://127.0.0.1:5000/v2/" in text + payload["avoids_docker_daemon_restart"] = "systemctl restart docker" not in text + payload["avoids_host_reboot"] = not re.search( + r"(^|[;&|[:space:]])reboot($|[;&|[:space:]])", + text, + ) + payload["executor_ready"] = all( + bool(payload[key]) + for key in ( + "has_check_mode", + "has_repair_once", + "has_v2_health_check", + "avoids_docker_daemon_restart", + "avoids_host_reboot", + ) + ) + return payload + + +def _status( + *, + registry_ready: bool, + candidate_packaged: bool, + upstream_502_shape: bool, + gateway_drift_shape: bool, +) -> str: + if registry_ready: + return "harbor_registry_ready" + if candidate_packaged and upstream_502_shape: + return "controlled_recovery_preflight_packaged_harbor_registry_upstream_502" + if candidate_packaged and gateway_drift_shape: + return "controlled_recovery_preflight_packaged_public_gateway_route_drift" + return "blocked_harbor_registry_recovery_evidence_incomplete" + + +def _safe_next_step( + *, + registry_ready: bool, + upstream_502_shape: bool, + gateway_drift_shape: bool, + candidate_packaged: bool, +) -> str: + if registry_ready: + return "rerun_gitea_cd_and_verify_deploy_marker" + if candidate_packaged and upstream_502_shape: + return ( + "run_harbor_watchdog_check_then_repair_once_with_public_" + "registry_v2_verifier" + ) + if candidate_packaged and gateway_drift_shape: + return ( + "prepare_188_registry_nginx_rendered_diff_check_mode_then_route_" + "smoke_without_provider_switch" + ) + return "rerun_public_and_internal_registry_v2_readback_before_apply" + + +def _active_blockers( + *, + registry_ready: bool, + route_target_ready: bool, + watchdog_executor_ready: bool, + upstream_502_shape: bool, + gateway_drift_shape: bool, + public_v2_status: Any, + internal_v2_status: Any, +) -> list[str]: + if registry_ready: + return [] + blockers = [ + f"public_registry_v2_http_{public_v2_status}", + f"internal_110_registry_v2_http_{internal_v2_status}", + ] + if not route_target_ready: + blockers.append("registry_public_gateway_route_source_target_missing") + if not watchdog_executor_ready: + blockers.append("harbor_watchdog_single_run_executor_contract_missing") + if upstream_502_shape: + blockers.append("harbor_registry_upstream_502") + if gateway_drift_shape: + blockers.append("registry_public_gateway_route_drift") + blockers.append("harbor_registry_control_channel_readback_required") + return _unique_strings([str(item) for item in blockers]) + + +def _source_diff_status( + *, + registry_ready: bool, + route_target_ready: bool, + upstream_502_shape: bool, + gateway_drift_shape: bool, +) -> str: + if registry_ready: + return "no_recovery_diff_required" + if not route_target_ready: + return "route_source_target_missing" + if upstream_502_shape: + return "route_source_present_110_harbor_registry_upstream_502" + if gateway_drift_shape: + return "route_source_present_public_gateway_route_drift" + return "route_source_present_reprobe_required" + + +def _current_apply_blocker( + *, + registry_ready: bool, + candidate_packaged: bool, + upstream_502_shape: bool, + gateway_drift_shape: bool, +) -> str: + if registry_ready: + return "" + if candidate_packaged and upstream_502_shape: + return "110_harbor_watchdog_control_channel_readback_required_before_repair_once" + if candidate_packaged and gateway_drift_shape: + return "188_nginx_rendered_diff_and_nginx_t_required_before_route_apply" + return "registry_v2_shape_and_executor_contract_required_before_apply" + + +def _controlled_work_items( + *, + candidate_packaged: bool, + upstream_502_shape: bool, + gateway_drift_shape: bool, +) -> list[dict[str, Any]]: + base_status = ( + "queued_waiting_control_channel_readback" + if candidate_packaged + else "blocked_waiting_registry_v2_evidence" + ) + return [ + { + "id": "harbor-registry-v2-readback", + "title": "Registry /v2 public and internal readback", + "owner_agent": "SRE Agent", + "status": "done" if candidate_packaged else "needs_reprobe", + "required_output": "public_and_internal_registry_v2_status_matrix", + }, + { + "id": "harbor-watchdog-check", + "title": "Harbor watchdog check-mode control readback", + "owner_agent": "DevOps Agent", + "status": base_status, + "required_output": "AWOOOI_HARBOR_WATCHDOG_CHECK_without_secret_or_docker_daemon_restart", + }, + { + "id": "harbor-watchdog-repair-once", + "title": "Harbor watchdog bounded repair-once", + "owner_agent": "DevOps Agent", + "status": ( + "waiting_check_mode_failure" + if upstream_502_shape + else "not_selected_for_current_shape" + ), + "required_output": "HARBOR_REPAIR_PERFORMED_receipt_and_registry_v2_verifier", + }, + { + "id": "registry-public-gateway-route-diff", + "title": "188 registry public gateway rendered diff", + "owner_agent": "Gateway Agent", + "status": ( + "waiting_rendered_diff_check_mode" + if gateway_drift_shape + else "not_selected_for_current_shape" + ), + "required_output": "nginx_t_and_route_smoke_evidence_no_provider_switch", + }, + { + "id": "harbor-registry-learning-writeback", + "title": "KM / RAG / MCP / PlayBook trust writeback", + "owner_agent": "Learning Agent", + "status": "ready", + "required_output": "metadata_only_registry_recovery_receipt_and_trust_delta", + }, + ] + + +def _controlled_action_candidates( + *, + candidate_packaged: bool, + upstream_502_shape: bool, + gateway_drift_shape: bool, +) -> list[dict[str, Any]]: + candidates: list[dict[str, Any]] = [ + { + "id": "harbor_watchdog_check", + "execution_route": "host_110_harbor_watchdog_executor", + "current_status": ( + "queued_waiting_control_channel_readback" + if candidate_packaged + else "blocked_waiting_registry_v2_evidence" + ), + "check_mode": "harbor_watchdog_check_registry_v2_probe", + "forbidden_actions": [ + "secret_read", + "docker_daemon_restart", + "host_reboot", + "node_drain", + "nginx_reload", + "database_restore", + ], + } + ] + if upstream_502_shape: + candidates.append( + { + "id": "harbor_watchdog_repair_once", + "execution_route": "host_110_harbor_watchdog_executor", + "current_status": "waiting_check_unhealthy_receipt", + "check_mode": "watchdog_check_then_single_repair_attempt", + "bounded_apply": ( + "run existing harbor-watchdog --repair-once only; no " + "Docker daemon restart, no host reboot, no provider switch" + ), + } + ) + if gateway_drift_shape: + candidates.append( + { + "id": "registry_public_gateway_rendered_diff", + "execution_route": "host_188_nginx_controlled_apply_executor", + "current_status": "waiting_rendered_diff_check_mode", + "check_mode": "nginx_rendered_diff_and_nginx_t", + "bounded_apply": ( + "apply only registry.wooo.work route source diff after " + "nginx -t and rollback backup; no provider switch" + ), + } + ) + return candidates + + +def _post_apply_verifiers() -> list[dict[str, Any]]: + return [ + { + "id": "public_registry_v2", + "url": "https://registry.wooo.work/v2/", + "expected_http_status": [200, 401], + }, + { + "id": "internal_110_registry_v2", + "url": "http://192.168.0.110:5000/v2/", + "expected_http_status": [200, 401], + }, + { + "id": "gitea_cd_rerun_readback", + "url": "/api/v1/agents/awoooi-priority-work-order-readback", + "expected_rollup": "production deploy marker advances after Harbor recovery", + }, + ] + + +def _learning_writeback_contracts() -> list[dict[str, Any]]: + targets = [ + "timeline_event", + "knowledge_entry", + "rag_embedding", + "mcp_tool_registry_signal", + "playbook_trust_score", + "controlled_recovery_report", + ] + return [ + { + "target": target, + "writeback_mode": "metadata_only", + "raw_log_allowed": False, + "secret_value_allowed": False, + "receipt_key": f"harbor_registry_recovery::{target}", + } + for target in targets + ] + + +def _probe_endpoint( + probe: Probe, + url: str, + timeout_seconds: float, +) -> dict[str, Any]: + try: + result = probe(url, timeout_seconds) + except Exception as exc: # pragma: no cover - defensive API boundary + result = {"http_status": 0, "error": exc.__class__.__name__} + return {"url": url, **_dict(result)} + + +def _http_probe(url: str, timeout_seconds: float) -> dict[str, Any]: + request = urllib.request.Request( + url, + headers={"User-Agent": "awoooi-harbor-registry-preflight/1.0"}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + response.read(512) + return {"http_status": int(getattr(response, "status", 200))} + except urllib.error.HTTPError as exc: + exc.read(512) + return {"http_status": int(exc.code)} + except urllib.error.URLError as exc: + return {"http_status": 0, "error": str(exc.reason)} + except TimeoutError: + return {"http_status": 0, "error": "timeout"} + + +def _registry_ready(status_code: Any) -> bool: + return status_code in {200, 401} + + +def _dict(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _strings(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if item is not None] + + +def _unique_strings(values: list[str]) -> list[str]: + seen: set[str] = set() + unique: list[str] = [] + for value in values: + if value in seen: + continue + seen.add(value) + unique.append(value) + return unique diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 09e9e6663..d377b19e8 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -10,10 +10,14 @@ from fastapi.testclient import TestClient from src.api.v1 import agents from src.api.v1.agents import router from src.services.awoooi_priority_work_order_readback import ( + apply_harbor_registry_controlled_recovery_preflight, apply_stockplatform_public_api_controlled_recovery_preflight, apply_stockplatform_public_api_runtime_readback, load_latest_awoooi_priority_work_order_readback, ) +from src.services.harbor_registry_controlled_recovery_preflight import ( + load_latest_harbor_registry_controlled_recovery_preflight, +) from src.services.stockplatform_public_api_controlled_recovery_preflight import ( load_latest_stockplatform_public_api_controlled_recovery_preflight, ) @@ -71,6 +75,11 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot( "load_latest_stockplatform_public_api_runtime_readback", _stockplatform_runtime_ready, ) + monkeypatch.setattr( + agents, + "load_latest_harbor_registry_controlled_recovery_preflight", + _harbor_registry_ready, + ) app = FastAPI() app.include_router(router, prefix="/api/v1") client = TestClient(app) @@ -88,6 +97,48 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot( assert "do not reboot" in data["next_execution_order"][0] +def test_awoooi_priority_work_order_readback_overlays_harbor_deploy_blocker( + tmp_path: Path, +): + payload = load_latest_awoooi_priority_work_order_readback() + recovery = load_latest_harbor_registry_controlled_recovery_preflight( + route_source_path=_harbor_route_source(tmp_path), + watchdog_source_path=_harbor_watchdog_source(tmp_path), + probe=_harbor_upstream_502_probe, + ) + + apply_harbor_registry_controlled_recovery_preflight(payload, recovery) + + state = payload["mainline_execution_state"] + in_progress = payload["in_progress_or_blocked_in_priority_order"][0] + evidence = in_progress["evidence"] + assert payload["status"] == ( + "p0_006_blocked_harbor_registry_controlled_recovery_preflight" + ) + assert state["active_p0_state"] == ( + "blocked_harbor_registry_controlled_recovery_preflight" + ) + assert state["next_executable_mainline_workplan_id"] == ( + "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT" + ) + assert evidence["harbor_registry_controlled_recovery_candidate_packaged"] is True + assert evidence["harbor_registry_upstream_502_shape"] is True + assert evidence["harbor_registry_watchdog_executor_ready"] is True + assert evidence["harbor_registry_runtime_write_gate"] == "controlled_after_preflight" + assert evidence["harbor_registry_current_apply_allowed"] is False + assert "not a manual end state" in in_progress["reason"] + assert "harbor-watchdog.sh --check" in in_progress["professional_fix"][ + "action" + ] + assert "Do not restart Docker daemon" in in_progress["professional_fix"]["action"] + assert payload["summary"][ + "harbor_registry_controlled_recovery_candidate_packaged" + ] is True + assert payload["next_execution_order"][0].startswith( + "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT" + ) + + def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift(): payload = load_latest_awoooi_priority_work_order_readback() @@ -367,3 +418,69 @@ def _stockplatform_runtime_data_dependency_blocked() -> dict: ], }, } + + +def _harbor_registry_ready() -> dict: + return { + "schema_version": "harbor_registry_controlled_recovery_preflight_v1", + "priority": "P0-006", + "scope": "harbor_registry_controlled_recovery_preflight", + "status": "harbor_registry_ready", + "safe_next_step": "rerun_gitea_cd_and_verify_deploy_marker", + "active_blockers": [], + "rollups": { + "registry_ready": True, + "public_registry_v2_ready": True, + "internal_110_registry_v2_ready": True, + "controlled_recovery_candidate_packaged": False, + }, + } + + +def _harbor_route_source(tmp_path: Path) -> Path: + path = tmp_path / "188-internal-tools-https.conf.j2" + path.write_text( + """ +server { + listen 443 ssl http2; + server_name harbor.wooo.work; + location / { + proxy_pass http://192.168.0.110:5000; + } +} +server { + listen 443 ssl http2; + server_name registry.wooo.work; + location / { + proxy_pass http://192.168.0.110:5000; + } +} +""", + encoding="utf-8", + ) + return path + + +def _harbor_watchdog_source(tmp_path: Path) -> Path: + path = tmp_path / "harbor-watchdog.sh" + path.write_text( + """ +harbor_is_healthy() { + curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ +} +case "${1:-}" in + --check) echo AWOOOI_HARBOR_WATCHDOG_CHECK ;; + --repair-once) echo HARBOR_REPAIR_PERFORMED=1 ;; +esac +docker compose up -d harbor-log +docker compose up -d +""", + encoding="utf-8", + ) + return path + + +def _harbor_upstream_502_probe(url: str, _timeout_seconds: float) -> dict: + if "/v2/" in url: + return {"http_status": 502} + return {"http_status": 200} diff --git a/apps/api/tests/test_harbor_registry_controlled_recovery_preflight.py b/apps/api/tests/test_harbor_registry_controlled_recovery_preflight.py new file mode 100644 index 000000000..559e2ae9b --- /dev/null +++ b/apps/api/tests/test_harbor_registry_controlled_recovery_preflight.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.api.v1 import agents +from src.api.v1.agents import router +from src.services.harbor_registry_controlled_recovery_preflight import ( + load_latest_harbor_registry_controlled_recovery_preflight, +) + + +def test_harbor_registry_preflight_packages_110_upstream_502(tmp_path: Path) -> None: + payload = load_latest_harbor_registry_controlled_recovery_preflight( + route_source_path=_route_source(tmp_path), + watchdog_source_path=_watchdog_source(tmp_path), + probe=_probe_for_upstream_502, + ) + + assert payload["schema_version"] == "harbor_registry_controlled_recovery_preflight_v1" + assert payload["priority"] == "P0-006" + assert payload["status"] == ( + "controlled_recovery_preflight_packaged_harbor_registry_upstream_502" + ) + assert payload["source_of_truth_diff"]["status"] == ( + "route_source_present_110_harbor_registry_upstream_502" + ) + assert payload["rollups"]["harbor_registry_upstream_502_shape"] is True + assert payload["rollups"]["controlled_recovery_candidate_packaged"] is True + assert payload["rollups"]["watchdog_executor_ready"] is True + assert payload["controlled_apply_policy"]["manual_end_state"] is False + assert payload["controlled_apply_policy"]["break_glass_required"] is False + assert payload["controlled_apply_policy"]["current_apply_allowed"] is False + assert payload["controlled_action_candidates"][1]["id"] == ( + "harbor_watchdog_repair_once" + ) + assert payload["controlled_work_items"][2]["id"] == "harbor-watchdog-repair-once" + assert "harbor_registry_upstream_502" in payload["active_blockers"] + assert "docker_daemon_restart" in payload["controlled_apply_policy"][ + "forbidden_scope" + ] + assert payload["operation_boundaries"]["ssh_used"] is False + assert payload["operation_boundaries"]["docker_command_performed"] is False + assert payload["operation_boundaries"]["docker_daemon_restart_performed"] is False + assert payload["operation_boundaries"]["host_reboot_performed"] is False + assert payload["operation_boundaries"]["secret_value_collection_allowed"] is False + + +def test_harbor_registry_preflight_ready_when_registry_v2_ready(tmp_path: Path) -> None: + payload = load_latest_harbor_registry_controlled_recovery_preflight( + route_source_path=_route_source(tmp_path), + watchdog_source_path=_watchdog_source(tmp_path), + probe=_probe_ready, + ) + + assert payload["status"] == "harbor_registry_ready" + assert payload["active_blockers"] == [] + assert payload["rollups"]["registry_ready"] is True + assert payload["safe_next_step"] == "rerun_gitea_cd_and_verify_deploy_marker" + + +def test_harbor_registry_preflight_routes_gateway_drift(tmp_path: Path) -> None: + payload = load_latest_harbor_registry_controlled_recovery_preflight( + route_source_path=_route_source(tmp_path), + watchdog_source_path=_watchdog_source(tmp_path), + probe=_probe_gateway_drift, + ) + + assert payload["status"] == ( + "controlled_recovery_preflight_packaged_public_gateway_route_drift" + ) + assert payload["source_of_truth_diff"]["status"] == ( + "route_source_present_public_gateway_route_drift" + ) + assert payload["rollups"]["public_gateway_route_drift_shape"] is True + assert payload["controlled_action_candidates"][1]["id"] == ( + "registry_public_gateway_rendered_diff" + ) + + +def test_harbor_registry_preflight_endpoint_returns_payload( + monkeypatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr( + agents, + "load_latest_harbor_registry_controlled_recovery_preflight", + lambda: load_latest_harbor_registry_controlled_recovery_preflight( + route_source_path=_route_source(tmp_path), + watchdog_source_path=_watchdog_source(tmp_path), + probe=_probe_for_upstream_502, + ), + ) + app = FastAPI() + app.include_router(router, prefix="/api/v1") + client = TestClient(app) + + response = client.get( + "/api/v1/agents/harbor-registry-controlled-recovery-preflight" + ) + + assert response.status_code == 200 + data = response.json() + assert data["status"] == ( + "controlled_recovery_preflight_packaged_harbor_registry_upstream_502" + ) + assert data["rollups"]["controlled_recovery_candidate_packaged"] is True + + +def _route_source(tmp_path: Path) -> Path: + path = tmp_path / "188-internal-tools-https.conf.j2" + path.write_text( + """ +server { + listen 443 ssl http2; + server_name harbor.wooo.work; + location / { + proxy_pass http://192.168.0.110:5000; + } +} +server { + listen 443 ssl http2; + server_name registry.wooo.work; + location / { + proxy_pass http://192.168.0.110:5000; + } +} +""", + encoding="utf-8", + ) + return path + + +def _watchdog_source(tmp_path: Path) -> Path: + path = tmp_path / "harbor-watchdog.sh" + path.write_text( + """ +harbor_is_healthy() { + curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ +} +case "${1:-}" in + --check) echo AWOOOI_HARBOR_WATCHDOG_CHECK ;; + --repair-once) echo HARBOR_REPAIR_PERFORMED=1 ;; +esac +docker compose up -d harbor-log +docker compose up -d +""", + encoding="utf-8", + ) + return path + + +def _probe_for_upstream_502(url: str, _timeout_seconds: float) -> dict: + if "/v2/" in url: + return {"http_status": 502} + return {"http_status": 200} + + +def _probe_ready(url: str, _timeout_seconds: float) -> dict: + if "/v2/" in url: + return {"http_status": 401} + return {"http_status": 200} + + +def _probe_gateway_drift(url: str, _timeout_seconds: float) -> dict: + if url == "https://registry.wooo.work/v2/": + return {"http_status": 502} + if url == "http://192.168.0.110:5000/v2/": + return {"http_status": 401} + return {"http_status": 200} diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index da602cfd5..43d6faa8e 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,24 @@ +## 2026-06-30 — 20:20 Harbor registry controlled recovery preflight 接入主線 + +**照主線修正的問題**: +- 新增 `harbor_registry_controlled_recovery_preflight` API service 與 `/api/v1/agents/harbor-registry-controlled-recovery-preflight`,把目前 Harbor / registry `/v2/` 502 轉成 AI controlled recovery package:target selector、188 route source diff、110 watchdog executor contract、bounded action candidates、post-apply verifier、KM / RAG / MCP / PlayBook metadata writeback contract。 +- `awoooi_priority_work_order_readback` 新增 Harbor overlay;當 registry `/v2/` public / internal 都不是 expected `200/401` 時,主線 next workplan 會先切到 `P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT`,不再只停在 StockPlatform 或人工終局。 +- 採用 Gitea main 已提交的 `harbor-watchdog.sh --check / --repair-once` 單次受控模式;本輪不保留舊 `--status-only` 命名,避免 executor contract 分裂。 +- Gitea CD controlled-runtime profile 納入新 Harbor preflight service / test / watchdog contract test,讓 P0 registry recovery 變更走窄 verifier。 + +**Live readback**: +- `registry.wooo.work/v2/` 與 `192.168.0.110:5000/v2/` 仍回 502;Harbor UI public / internal 皆為 200;本地 preflight 回 `controlled_recovery_preflight_packaged_harbor_registry_upstream_502`、`controlled_recovery_candidate_packaged=true`。 +- 本地 priority overlay 回 `p0_006_blocked_harbor_registry_controlled_recovery_preflight`,下一步為 watchdog check-mode → bounded repair-once → registry `/v2/` verifier → CD deploy marker readback。 +- Public Gitea latest visible CD `#4047` 正在跑 `90647f294 fix(recovery): add controlled harbor repair mode`;public jobs API head_sha 仍 stale,visible run 為 source truth。 + +**驗證**: +- `py_compile`、`ruff check`、`bash -n harbor-watchdog.sh` 通過。 +- focused pytest:`42 passed`。 +- controlled-runtime profile pytest:`340 passed`。 +- `guard-gitea-runner-pressure.py`:`auto_branch_events_on_110=0`、`generic_runner_labels=0`;`check-gitea-step-env-secrets.js` 通過;`git diff --check` 通過。 + +**邊界**:未讀 secret / token / `.env` / raw sessions / SQLite / auth;未使用 GitHub / `gh` / GitHub API;未 workflow_dispatch,沒有 SSH 成功讀取,沒有 Docker / Nginx / K3s / DB / firewall runtime 寫入,沒有重啟、node drain、provider route 切換或 force push。 + ## 2026-06-30 — 20:16 Public Gitea CD Harbor blocker classifier **照主線修正的問題**: diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 684ae81e9..9ddd0e932 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -361,16 +361,20 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() - "apps/api/src/services/reboot_auto_recovery_drill_preflight.py)", "apps/api/src/services/stockplatform_public_api_runtime_readback.py)", "apps/api/src/services/stockplatform_public_api_controlled_recovery_preflight.py)", + "apps/api/src/services/harbor_registry_controlled_recovery_preflight.py)", "apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py)", "apps/api/tests/test_stockplatform_public_api_runtime_readback.py)", "apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py)", + "apps/api/tests/test_harbor_registry_controlled_recovery_preflight.py)", "src/services/reboot_auto_recovery_slo_scorecard.py", "src/services/reboot_auto_recovery_drill_preflight.py", "src/services/stockplatform_public_api_runtime_readback.py", "src/services/stockplatform_public_api_controlled_recovery_preflight.py", + "src/services/harbor_registry_controlled_recovery_preflight.py", "tests/test_reboot_auto_recovery_slo_scorecard_api.py", "tests/test_stockplatform_public_api_runtime_readback.py", "tests/test_stockplatform_public_api_controlled_recovery_preflight.py", + "tests/test_harbor_registry_controlled_recovery_preflight.py", "scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service)", "scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer)", "scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh)", @@ -379,12 +383,14 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() - "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py)", "scripts/reboot-recovery/full-stack-cold-start-check.sh)", "scripts/reboot-recovery/full-stack-recovery-scorecard.sh)", + "scripts/reboot-recovery/harbor-watchdog.sh)", "scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh)", "scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh)", "scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh)", "scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py)", "scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py)", "scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py)", + "scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py)", "../../scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py", "../../scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py", "../../scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py", @@ -406,11 +412,13 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh)", "scripts/reboot-recovery/full-stack-cold-start-check.sh)", "scripts/reboot-recovery/full-stack-recovery-scorecard.sh)", + "scripts/reboot-recovery/harbor-watchdog.sh)", "scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh)", "scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py)", "scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py)", "scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py)", "scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py)", + "scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py)", "../../scripts/ops/backup-health-textfile-exporter.py", "../../scripts/backup/gitea-repo-bundle-backup.sh", "../../ops/monitoring/alerts-unified.yml", @@ -418,6 +426,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "../../scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh", "../../scripts/reboot-recovery/full-stack-cold-start-check.sh", "../../scripts/reboot-recovery/full-stack-recovery-scorecard.sh", + "../../scripts/reboot-recovery/harbor-watchdog.sh", "../../scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh", "../../scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh", "../../scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh", @@ -425,6 +434,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "../../scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py", "../../scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py", "../../scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py", + "../../scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py", ] for source in expected_sources: assert source in text diff --git a/scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py b/scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py new file mode 100644 index 000000000..e257c8ba7 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py @@ -0,0 +1,33 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +WATCHDOG = ROOT / "scripts/reboot-recovery/harbor-watchdog.sh" + + +def test_harbor_watchdog_exposes_single_run_contracts() -> None: + text = WATCHDOG.read_text(encoding="utf-8") + + assert "--check" in text + assert "--repair-once" in text + assert "AWOOOI_HARBOR_WATCHDOG_CHECK" in text + assert "mode=check" in text + assert "docker compose up -d harbor-log" in text + assert "docker compose up -d" in text + + +def test_harbor_watchdog_does_not_use_incident_grade_actions() -> None: + text = WATCHDOG.read_text(encoding="utf-8") + forbidden = [ + "systemctl restart docker", + "service docker restart", + "\nreboot", + "\nsudo reboot", + "\nshutdown", + "\nsudo shutdown", + "docker system prune", + "docker volume rm", + ] + + for pattern in forbidden: + assert pattern not in text