diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index 554681c2c..50c460bbd 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -486,6 +486,9 @@ from src.services.telegram_alert_monitoring_live_receipt_apply import ( from src.services.windows99_vmx_source_locator_readback import ( load_latest_windows99_vmx_source_locator_readback, ) +from src.services.windows99_vmx_source_backup_search_package import ( + load_latest_windows99_vmx_source_backup_search_package, +) from src.services.windows99_vmware_autostart_check_mode_package import ( load_latest_windows99_vmware_autostart_check_mode_package, ) @@ -1644,6 +1647,40 @@ async def get_windows99_vmx_source_locator_readback() -> dict[str, Any]: ) from exc +@router.get( + "/windows99-vmx-source-backup-search-package", + response_model=dict[str, Any], + summary="取得 Windows99 VMX source backup-search check-mode package", + description=( + "讀取 P0-006 Windows99 VMX source locator 與 backup/source search receipt，" + "產生 no-secret check-mode package。此端點不 SSH、不讀 VMX 內容、" + "不回傳原始 VMX 路徑、不啟停 VM、不寫 Windows、不改 registry/task、" + "不重啟主機、不讀 secret。" + ), +) +async def get_windows99_vmx_source_backup_search_package() -> dict[str, Any]: + """回傳 Windows99 VMX source backup/source search check-mode package。""" + try: + payload = await asyncio.to_thread( + load_latest_windows99_vmx_source_backup_search_package + ) + return redact_public_lan_topology(payload) + except FileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except (json.JSONDecodeError, ValueError) as exc: + logger.error( + "windows99_vmx_source_backup_search_package_invalid", + error=str(exc), + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Windows99 VMX source backup-search package 無效", + ) from exc + + @router.get( "/reboot-auto-recovery-drill-preflight", response_model=dict[str, Any], diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index 6a104a5f2..4900a4b84 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -2987,6 +2987,9 @@ def _enrich_from_current_readbacks( from src.services.windows99_vmx_source_locator_readback import ( build_windows99_vmx_source_locator_readback, ) + from src.services.windows99_vmx_source_backup_search_package import ( + build_windows99_vmx_source_backup_search_package, + ) from src.services.windows99_vmware_autostart_check_mode_package import ( build_windows99_vmware_autostart_check_mode_package, ) @@ -3044,6 +3047,12 @@ def _enrich_from_current_readbacks( repair_package=windows99_vmx_source_repair_package, ) ) + windows99_vmx_source_backup_search_package = ( + build_windows99_vmx_source_backup_search_package( + reboot_slo, + locator_readback=windows99_vmx_source_locator_readback, + ) + ) windows99_vmx_source_repair_action_chain = _dict( windows99_vmx_source_repair_package.get("action_chain") ) @@ -3818,6 +3827,74 @@ def _enrich_from_current_readbacks( state["windows99_vmx_source_locator_vm_power_change_performed"] = bool( windows99_vmx_source_locator_readback.get("vm_power_change_performed") is True ) + state["windows99_vmx_source_backup_search_package_status"] = str( + windows99_vmx_source_backup_search_package.get("status") or "unknown" + ) + state["windows99_vmx_source_backup_search_package_ready"] = bool( + windows99_vmx_source_backup_search_package.get("package_ready") is True + ) + state["windows99_vmx_source_backup_search_target_aliases"] = _strings( + windows99_vmx_source_backup_search_package.get("target_vm_aliases") + ) + state["windows99_vmx_source_backup_search_next_executable_step"] = str( + windows99_vmx_source_backup_search_package.get("next_executable_step") or "" + ) + state["windows99_vmx_source_backup_search_safe_next_step"] = str( + windows99_vmx_source_backup_search_package.get("safe_next_step") or "" + ) + state["windows99_vmx_source_backup_search_receipt_readback_present"] = bool( + windows99_vmx_source_backup_search_package.get("receipt_readback_present") + is True + ) + state["windows99_vmx_source_backup_search_host_count"] = _int( + windows99_vmx_source_backup_search_package.get("search_host_count") + ) + state["windows99_vmx_source_backup_search_searched_host_count"] = _int( + windows99_vmx_source_backup_search_package.get("searched_host_count") + ) + state["windows99_vmx_source_backup_search_blocked_host_count"] = _int( + windows99_vmx_source_backup_search_package.get("blocked_host_count") + ) + state["windows99_vmx_source_backup_search_pending_host_count"] = _int( + windows99_vmx_source_backup_search_package.get("pending_host_count") + ) + state["windows99_vmx_source_backup_search_blocked_host_aliases"] = _strings( + windows99_vmx_source_backup_search_package.get("blocked_host_aliases") + ) + state["windows99_vmx_source_backup_search_candidate_source_count"] = _int( + windows99_vmx_source_backup_search_package.get("candidate_source_count") + ) + state["windows99_vmx_source_backup_search_candidate_fingerprint_count"] = _int( + windows99_vmx_source_backup_search_package.get( + "candidate_source_fingerprint_count" + ) + ) + state["windows99_vmx_source_backup_search_candidate_source_found"] = bool( + windows99_vmx_source_backup_search_package.get("candidate_source_found") + is True + ) + state["windows99_vmx_source_backup_search_apply_allowed_by_this_package"] = bool( + windows99_vmx_source_backup_search_package.get("apply_allowed_by_this_package") + is True + ) + state["windows99_vmx_source_backup_search_raw_path_output"] = bool( + windows99_vmx_source_backup_search_package.get("raw_path_output") is True + ) + state["windows99_vmx_source_backup_search_path_fingerprint_only"] = bool( + windows99_vmx_source_backup_search_package.get("path_fingerprint_only") + is True + ) + state["windows99_vmx_source_backup_search_file_content_read"] = bool( + windows99_vmx_source_backup_search_package.get("file_content_read") is True + ) + state["windows99_vmx_source_backup_search_remote_write_performed"] = bool( + windows99_vmx_source_backup_search_package.get("remote_write_performed") + is True + ) + state["windows99_vmx_source_backup_search_vm_power_change_performed"] = bool( + windows99_vmx_source_backup_search_package.get("vm_power_change_performed") + is True + ) state["stale_snapshot_or_old_cd_runs_must_not_reopen_closed_work"] = True state["p0_004_template_copy_apply_gate_production_http_status"] = 200 state["p0_004_template_copy_apply_gate_runtime_readback_state"] = ( @@ -4404,6 +4481,66 @@ def _enrich_from_current_readbacks( evidence["windows99_vmx_source_locator_vm_power_change_performed"] = state[ "windows99_vmx_source_locator_vm_power_change_performed" ] + evidence["windows99_vmx_source_backup_search_package_status"] = state[ + "windows99_vmx_source_backup_search_package_status" + ] + evidence["windows99_vmx_source_backup_search_package_ready"] = state[ + "windows99_vmx_source_backup_search_package_ready" + ] + evidence["windows99_vmx_source_backup_search_target_aliases"] = state[ + "windows99_vmx_source_backup_search_target_aliases" + ] + evidence["windows99_vmx_source_backup_search_next_executable_step"] = state[ + "windows99_vmx_source_backup_search_next_executable_step" + ] + evidence["windows99_vmx_source_backup_search_safe_next_step"] = state[ + "windows99_vmx_source_backup_search_safe_next_step" + ] + evidence["windows99_vmx_source_backup_search_receipt_readback_present"] = ( + state["windows99_vmx_source_backup_search_receipt_readback_present"] + ) + evidence["windows99_vmx_source_backup_search_host_count"] = state[ + "windows99_vmx_source_backup_search_host_count" + ] + evidence["windows99_vmx_source_backup_search_searched_host_count"] = state[ + "windows99_vmx_source_backup_search_searched_host_count" + ] + evidence["windows99_vmx_source_backup_search_blocked_host_count"] = state[ + "windows99_vmx_source_backup_search_blocked_host_count" + ] + evidence["windows99_vmx_source_backup_search_pending_host_count"] = state[ + "windows99_vmx_source_backup_search_pending_host_count" + ] + evidence["windows99_vmx_source_backup_search_blocked_host_aliases"] = state[ + "windows99_vmx_source_backup_search_blocked_host_aliases" + ] + evidence["windows99_vmx_source_backup_search_candidate_source_count"] = state[ + "windows99_vmx_source_backup_search_candidate_source_count" + ] + evidence[ + "windows99_vmx_source_backup_search_candidate_fingerprint_count" + ] = state["windows99_vmx_source_backup_search_candidate_fingerprint_count"] + evidence["windows99_vmx_source_backup_search_candidate_source_found"] = state[ + "windows99_vmx_source_backup_search_candidate_source_found" + ] + evidence[ + "windows99_vmx_source_backup_search_apply_allowed_by_this_package" + ] = state["windows99_vmx_source_backup_search_apply_allowed_by_this_package"] + evidence["windows99_vmx_source_backup_search_raw_path_output"] = state[ + "windows99_vmx_source_backup_search_raw_path_output" + ] + evidence["windows99_vmx_source_backup_search_path_fingerprint_only"] = state[ + "windows99_vmx_source_backup_search_path_fingerprint_only" + ] + evidence["windows99_vmx_source_backup_search_file_content_read"] = state[ + "windows99_vmx_source_backup_search_file_content_read" + ] + evidence["windows99_vmx_source_backup_search_remote_write_performed"] = state[ + "windows99_vmx_source_backup_search_remote_write_performed" + ] + evidence[ + "windows99_vmx_source_backup_search_vm_power_change_performed" + ] = state["windows99_vmx_source_backup_search_vm_power_change_performed"] evidence["drill_preflight_status"] = str(reboot_preflight.get("status") or "") evidence["drill_preflight_ready"] = ( reboot_preflight_rollups.get("preflight_ready") is True @@ -4684,11 +4821,29 @@ def _refresh_rollups_after_stockplatform_overlay( rollups.get("active_p0_event_gated_by_fresh_reboot_window_only") is True and not stock_blocked ) + rollups["windows99_vmx_source_backup_search_status"] = str( + state.get("windows99_vmx_source_backup_search_package_status") or "unknown" + ) + rollups["windows99_vmx_source_backup_search_candidate_source_count"] = _int( + state.get("windows99_vmx_source_backup_search_candidate_source_count") + ) + rollups["windows99_vmx_source_backup_search_blocked_host_count"] = _int( + state.get("windows99_vmx_source_backup_search_blocked_host_count") + ) summary = _dict(payload.setdefault("summary", {})) summary["status"] = str(payload.get("status") or "") summary["active_p0_state"] = str(state.get("active_p0_state") or "") summary["active_p0_live_active_blockers"] = active_blockers + summary["windows99_vmx_source_backup_search_status"] = str( + state.get("windows99_vmx_source_backup_search_package_status") or "unknown" + ) + summary["windows99_vmx_source_backup_search_candidate_source_count"] = _int( + state.get("windows99_vmx_source_backup_search_candidate_source_count") + ) + summary["windows99_vmx_source_backup_search_safe_next_step"] = str( + state.get("windows99_vmx_source_backup_search_safe_next_step") or "" + ) summary["stockplatform_public_api_runtime_status"] = str( state.get("stockplatform_public_api_runtime_status") or "unknown" ) @@ -5309,6 +5464,33 @@ def _set_rollups_and_summary( state.get("windows99_vmx_source_locator_vm_power_change_performed") is True ), + "windows99_vmx_source_backup_search_status": str( + state.get("windows99_vmx_source_backup_search_package_status") + or "unknown" + ), + "windows99_vmx_source_backup_search_package_ready": ( + state.get("windows99_vmx_source_backup_search_package_ready") is True + ), + "windows99_vmx_source_backup_search_candidate_source_count": _int( + state.get("windows99_vmx_source_backup_search_candidate_source_count") + ), + "windows99_vmx_source_backup_search_blocked_host_count": _int( + state.get("windows99_vmx_source_backup_search_blocked_host_count") + ), + "windows99_vmx_source_backup_search_apply_allowed_by_this_package": ( + state.get( + "windows99_vmx_source_backup_search_apply_allowed_by_this_package" + ) + is True + ), + "windows99_vmx_source_backup_search_remote_write_performed": ( + state.get("windows99_vmx_source_backup_search_remote_write_performed") + is True + ), + "windows99_vmx_source_backup_search_vm_power_change_performed": ( + state.get("windows99_vmx_source_backup_search_vm_power_change_performed") + is True + ), "p0_004_runtime_readback_ready": p0_004_ready, "reboot_drill_preflight_runtime_readback_ready": ( state.get("reboot_drill_preflight_runtime_readback_state") == "ready" @@ -5744,6 +5926,36 @@ def _set_rollups_and_summary( state.get("windows99_vmx_source_locator_vm_power_change_performed") is True ), + "windows99_vmx_source_backup_search_status": str( + state.get("windows99_vmx_source_backup_search_package_status") + or "unknown" + ), + "windows99_vmx_source_backup_search_package_ready": ( + state.get("windows99_vmx_source_backup_search_package_ready") is True + ), + "windows99_vmx_source_backup_search_candidate_source_count": _int( + state.get("windows99_vmx_source_backup_search_candidate_source_count") + ), + "windows99_vmx_source_backup_search_blocked_host_count": _int( + state.get("windows99_vmx_source_backup_search_blocked_host_count") + ), + "windows99_vmx_source_backup_search_safe_next_step": str( + state.get("windows99_vmx_source_backup_search_safe_next_step") or "" + ), + "windows99_vmx_source_backup_search_apply_allowed_by_this_package": ( + state.get( + "windows99_vmx_source_backup_search_apply_allowed_by_this_package" + ) + is True + ), + "windows99_vmx_source_backup_search_remote_write_performed": ( + state.get("windows99_vmx_source_backup_search_remote_write_performed") + is True + ), + "windows99_vmx_source_backup_search_vm_power_change_performed": ( + state.get("windows99_vmx_source_backup_search_vm_power_change_performed") + is True + ), "windows99_winrm_http_open": ( state.get("windows99_winrm_http_open") is True ), diff --git a/apps/api/src/services/windows99_vmx_source_backup_search_package.py b/apps/api/src/services/windows99_vmx_source_backup_search_package.py new file mode 100644 index 000000000..53306f40c --- /dev/null +++ b/apps/api/src/services/windows99_vmx_source_backup_search_package.py @@ -0,0 +1,323 @@ +"""Windows99 VMX source backup-search check-mode package. + +This readback turns "111 VMX source not found" into a fixed no-secret search +contract across backup/source hosts. It does not SSH from the API, read VMX +contents, expose raw paths, write Windows state, start VMs, stop VMs, restart +services, or reboot hosts. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from src.services.reboot_auto_recovery_slo_scorecard import ( + _taipei_now_iso, + load_latest_reboot_auto_recovery_slo_scorecard, +) +from src.services.snapshot_paths import default_operations_dir +from src.services.windows99_vmx_source_locator_readback import ( + build_windows99_vmx_source_locator_readback, +) +from src.services.windows99_vmx_source_repair_package import ( + build_windows99_vmx_source_repair_package, +) + +_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__)) +_SCHEMA_VERSION = "windows99_vmx_source_backup_search_package_v1" +_RECEIPT_SCHEMA_VERSION = "windows99_vmx_source_backup_search_receipt_v1" +_RECEIPT_FILE = "windows99-vmx-source-backup-search-receipt.snapshot.json" +_COLLECTOR_SCRIPT_PATH = ( + "scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh" +) +_DEFAULT_SEARCH_HOSTS = ["99", "110", "120", "121", "188", "112", "local"] +_FORBIDDEN_ACTIONS = [ + "read_windows_password_or_secret", + "read_vmx_file_contents", + "emit_raw_internal_paths_on_public_endpoint", + "vm_power_change", + "vmrun_start_or_stop", + "host_reboot", + "windows_service_restart", + "windows_registry_apply", + "scheduled_task_register_or_modify", + "docker_nginx_k3s_db_firewall_restart", + "destructive_restore", +] + + +def _dict(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _list(value: Any) -> list[Any]: + return value if isinstance(value, list) else [] + + +def _strings(value: Any) -> list[str]: + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + if isinstance(value, str): + return [part.strip() for part in value.split(",") if part.strip()] + return [] + + +def _int(value: Any, default: int = 0) -> int: + if isinstance(value, bool): + return default + if isinstance(value, int): + return value + if isinstance(value, str): + try: + return int(value.strip()) + except ValueError: + return default + return default + + +def _load_backup_search_receipt(operations_dir: Path | None = None) -> dict[str, Any]: + path = (operations_dir or _DEFAULT_OPERATIONS_DIR) / _RECEIPT_FILE + if not path.exists(): + return {} + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{path}: expected JSON object") + if payload.get("schema_version") != _RECEIPT_SCHEMA_VERSION: + raise ValueError(f"{path}: unexpected schema_version") + serialized = json.dumps(payload, ensure_ascii=False) + forbidden = ["192.168.0.", "D:\\", "C:\\", "E:\\", "/Users/", "/Volumes/"] + leaked = [fragment for fragment in forbidden if fragment in serialized] + if leaked: + raise ValueError(f"{path}: unsafe backup-search receipt fragments: {leaked}") + return payload + + +def _receipt_applies(receipt: dict[str, Any], target_aliases: list[str]) -> bool: + receipt_aliases = _strings(receipt.get("target_vm_aliases")) + return bool( + receipt.get("collector_readback_present") is True + and receipt.get("target_host_alias") == "99" + and receipt_aliases + and set(receipt_aliases).issubset(set(target_aliases or receipt_aliases)) + ) + + +def _default_host_rows(locator_readback: dict[str, Any]) -> list[dict[str, Any]]: + locator_candidate_count = _int(locator_readback.get("candidate_source_count"), 0) + locator_known = locator_readback.get("candidate_source_count_known") is True + rows = [ + { + "host_alias": "99", + "search_role": "windows_vmware_roots_via_locator", + "status": str(locator_readback.get("locate_status") or "not_collected"), + "candidate_source_count": locator_candidate_count if locator_known else 0, + "candidate_source_count_known": locator_known, + "collector": "windows99-vmx-source-locator-readback", + } + ] + for alias in ["110", "120", "121", "188", "112", "local"]: + rows.append( + { + "host_alias": alias, + "search_role": "linux_or_local_backup_roots", + "status": "check_mode_pending", + "candidate_source_count": 0, + "candidate_source_count_known": False, + "collector": _COLLECTOR_SCRIPT_PATH, + } + ) + return rows + + +def _host_rows_from_receipt( + receipt: dict[str, Any], + locator_readback: dict[str, Any], +) -> list[dict[str, Any]]: + rows = _list(receipt.get("host_rows")) + if rows: + return [dict(row) for row in rows if isinstance(row, dict)] + return _default_host_rows(locator_readback) + + +def build_windows99_vmx_source_backup_search_package( + scorecard: dict[str, Any], + *, + locator_readback: dict[str, Any] | None = None, + collector_receipt: dict[str, Any] | None = None, + operations_dir: Path | None = None, + generated_at: str | None = None, +) -> dict[str, Any]: + repair_package = build_windows99_vmx_source_repair_package(scorecard) + locator = locator_readback or build_windows99_vmx_source_locator_readback( + scorecard, + repair_package=repair_package, + operations_dir=operations_dir, + ) + missing_aliases = _strings(repair_package.get("missing_vmx_aliases")) + target_aliases = missing_aliases or _strings(locator.get("target_vm_aliases")) + receipt = ( + _dict(collector_receipt) + if collector_receipt is not None + else _load_backup_search_receipt(operations_dir) + ) + receipt_applies = _receipt_applies(receipt, target_aliases) + host_rows = ( + _host_rows_from_receipt(receipt, locator) + if receipt_applies + else _default_host_rows(locator) + ) + searched_rows = [ + row for row in host_rows if row.get("status") in {"searched", "not_matched"} + ] + blocked_rows = [ + row + for row in host_rows + if str(row.get("status") or "").startswith("blocked_") + or row.get("status") == "transport_intermittent" + ] + pending_rows = [ + row + for row in host_rows + if row.get("candidate_source_count_known") is not True + and row not in blocked_rows + ] + candidate_count = sum(_int(row.get("candidate_source_count"), 0) for row in host_rows) + candidate_fingerprint_count = sum( + _int(row.get("candidate_source_fingerprint_count"), 0) for row in host_rows + ) + locator_candidate_count = _int(locator.get("candidate_source_count"), 0) + locator_no_candidate = bool( + locator.get("candidate_source_count_known") is True + and locator_candidate_count == 0 + ) + search_required = bool(missing_aliases) + + if not search_required: + status = "ready_no_windows99_vmx_backup_search_required" + next_step = "rerun_no_secret_vmware_verify_collector" + safe_next_step = "rerun_no_secret_collector_and_reboot_slo_scorecard_verify_only" + elif candidate_count > 0: + status = "candidate_fingerprint_found_requires_internal_path_resolution" + next_step = "resolve_candidate_fingerprint_to_authorized_internal_path" + safe_next_step = ( + "resolve_candidate_fingerprint_to_authorized_internal_path_then_" + "build_scoped_relink_dry_run_no_write" + ) + elif receipt_applies and blocked_rows: + status = "blocked_windows99_vmx_backup_source_not_found_some_hosts_unsearched" + next_step = "rerun_backup_search_for_blocked_hosts_or_offline_inventory" + safe_next_step = ( + "rerun_windows99_vmx_source_backup_search_check_mode_for_blocked_hosts_" + "then_update_receipt_no_write" + ) + elif receipt_applies: + status = "blocked_windows99_vmx_backup_source_not_found" + next_step = "continue_verified_backup_inventory_search_or_console_source_path" + safe_next_step = ( + "recover_verified_external_backup_artifact_or_authorized_console_path_" + "then_build_scoped_relink_dry_run_no_write" + ) + elif locator_no_candidate: + status = "check_mode_package_ready_windows99_vmx_backup_search_required" + next_step = "run_windows99_vmx_source_backup_search_check_mode" + safe_next_step = ( + "bash scripts/reboot-recovery/collect-windows99-vmx-source-backup-" + "search.sh --timeout 35" + ) + else: + status = "check_mode_package_ready_windows99_vmx_identity_or_backup_search_required" + next_step = "run_locator_confirmation_or_backup_search_check_mode" + safe_next_step = ( + "confirm_locator_candidate_identity_or_run_backup_search_check_mode_" + "before_scoped_relink_dry_run" + ) + + return { + "schema_version": _SCHEMA_VERSION, + "generated_at": generated_at + or str(scorecard.get("generated_at") or _taipei_now_iso()), + "status": status, + "package_ready": search_required, + "target_host_alias": "99", + "target_vm_aliases": target_aliases, + "missing_vmx_aliases": missing_aliases, + "next_executable_step": next_step, + "safe_next_step": safe_next_step, + "collector_script_path": _COLLECTOR_SCRIPT_PATH, + "collector_schema_version": "windows99_vmx_source_backup_search_collector_v1", + "collector_command": ( + "bash scripts/reboot-recovery/collect-windows99-vmx-source-backup-" + "search.sh --timeout 35" + ), + "receipt_readback_present": receipt_applies, + "receipt_ref": str(receipt.get("receipt_ref") or "") if receipt_applies else "", + "receipt_generated_at": str(receipt.get("generated_at") or "") + if receipt_applies + else "", + "source_locator_status": str(locator.get("status") or "unknown"), + "source_locator_candidate_source_count": locator_candidate_count, + "source_locator_candidate_source_count_known": bool( + locator.get("candidate_source_count_known") is True + ), + "search_host_aliases": _DEFAULT_SEARCH_HOSTS, + "host_rows": host_rows, + "search_host_count": len(host_rows), + "searched_host_count": len(searched_rows), + "blocked_host_count": len(blocked_rows), + "pending_host_count": len(pending_rows), + "blocked_host_aliases": _strings( + [row.get("host_alias") for row in blocked_rows] + ), + "candidate_source_count": candidate_count, + "candidate_source_fingerprint_count": candidate_fingerprint_count, + "candidate_source_found": candidate_count > 0, + "internal_path_resolution_required": candidate_count > 0, + "public_lan_topology_redacted": True, + "redacted_public_display_only": True, + "raw_path_output": False, + "path_fingerprint_only": True, + "file_content_read": False, + "secret_value_read": False, + "password_prompt_allowed": False, + "remote_write_performed": False, + "host_reboot_performed": False, + "vm_power_change_performed": False, + "windows_service_restart_performed": False, + "windows_registry_apply_performed": False, + "scheduled_task_write_performed": False, + "vm_power_change_allowed": False, + "destructive_restore_allowed": False, + "apply_allowed_by_this_package": False, + "forbidden_actions": _FORBIDDEN_ACTIONS, + "operation_boundaries": { + "secret_value_read": False, + "password_prompt_allowed": False, + "file_content_read": False, + "raw_path_output": False, + "path_fingerprint_only": True, + "remote_write_performed": False, + "host_reboot_performed": False, + "vm_power_change_performed": False, + "windows_service_restart_performed": False, + "windows_registry_apply_performed": False, + "scheduled_task_write_performed": False, + }, + "post_search_verifier": [ + "GET /api/v1/agents/windows99-vmx-source-backup-search-package", + "GET /api/v1/agents/windows99-vmx-source-locator-readback", + "GET /api/v1/agents/reboot-auto-recovery-slo-scorecard", + "GET /api/v1/agents/awoooi-priority-work-order-readback", + ], + "km_writeback_ref": "km://awoooi/reboot-slo/windows99-vmx-source-backup-search-111", + "playbook_trust_writeback_ref": ( + "playbook://awoooi/windows99-vmx-source-backup-search/" + "windows99-vmx-source-backup-search-111" + ), + } + + +def load_latest_windows99_vmx_source_backup_search_package() -> dict[str, Any]: + return build_windows99_vmx_source_backup_search_package( + load_latest_reboot_auto_recovery_slo_scorecard() + ) diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 8ae41f987..db5adc95b 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -812,7 +812,7 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a is False ) assert state["windows99_vmx_source_locator_status"] == ( - "collector_readback_verified_source_required_before_relink_dry_run" + "collector_readback_no_scoped_relink_candidate" ) assert state["windows99_vmx_source_locator_required"] is True assert state["windows99_vmx_source_locator_check_mode_ready"] is True @@ -826,19 +826,19 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a ) assert state["windows99_vmx_source_locator_target_aliases"] == ["111"] assert state["windows99_vmx_source_locator_next_executable_step"] == ( - "provide_verified_vmx_source_or_identity_evidence_then_scoped_relink_dry_run" + "provide_existing_vmx_source_or_backup_path" ) assert state["windows99_vmx_source_locator_expected_vmx_path_count"] == 1 - assert state["windows99_vmx_source_locator_candidate_source_count"] == 6 + assert state["windows99_vmx_source_locator_candidate_source_count"] == 0 assert state["windows99_vmx_source_locator_alias_hint_candidate_count"] == 0 - assert state["windows99_vmx_source_locator_unassigned_ubuntu_candidate_count"] == 5 + assert state["windows99_vmx_source_locator_unassigned_ubuntu_candidate_count"] == 0 assert ( state[ "windows99_vmx_source_locator_identity_unassigned_ubuntu_candidate_count" ] - == 1 + == 0 ) - assert state["windows99_vmx_source_locator_candidate_confirmation_required"] is True + assert state["windows99_vmx_source_locator_candidate_confirmation_required"] is False assert ( state[ "windows99_vmx_source_locator_human_confirmation_required_for_dry_run" @@ -849,7 +849,7 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a state[ "windows99_vmx_source_locator_verified_source_or_identity_evidence_required" ] - is True + is False ) assert ( state["windows99_vmx_source_locator_controlled_relink_dry_run_ready"] @@ -857,12 +857,12 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a ) assert state[ "windows99_vmx_source_locator_controlled_relink_dry_run_blocker" - ] == "verified_vmx_source_or_identity_evidence_missing" + ] == "scoped_relink_candidate_not_verified" assert state["windows99_vmx_source_locator_relink_dry_run_plan_id"] == ( "windows99-vmx-source-relink-dry-run-111" ) assert state["windows99_vmx_source_locator_candidate_confirmation_status"] == ( - "blocked_single_unassigned_ubuntu_candidate_identity_not_confirmed" + "blocked_no_candidate_source_found" ) assert state["windows99_vmx_source_locator_candidate_identity_decision"] == ( "blocked" @@ -876,12 +876,7 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a assert state["windows99_vmx_source_locator_candidate_relink_dry_run_ready"] is False assert state[ "windows99_vmx_source_locator_candidate_confirmation_missing_evidence" - ] == [ - "target_alias_identity_hint", - "verified_backup_or_authorized_console_candidate_identity", - "pre_apply_backup_plan", - "scoped_relink_dry_run", - ] + ] == ["existing_vmx_source_or_verified_backup_path"] assert ( state[ "windows99_vmx_source_locator_apply_allowed_by_candidate_confirmation_gate" @@ -890,9 +885,36 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a ) assert state["windows99_vmx_source_locator_candidate_source_count_known"] is True assert state["windows99_vmx_source_locator_collector_readback_present"] is True - assert state["windows99_vmx_source_locator_file_content_read"] is True + assert state["windows99_vmx_source_locator_file_content_read"] is False assert state["windows99_vmx_source_locator_remote_write_performed"] is False assert state["windows99_vmx_source_locator_vm_power_change_performed"] is False + assert state["windows99_vmx_source_backup_search_package_status"] == ( + "blocked_windows99_vmx_backup_source_not_found_some_hosts_unsearched" + ) + assert state["windows99_vmx_source_backup_search_package_ready"] is True + assert state["windows99_vmx_source_backup_search_target_aliases"] == ["111"] + assert state["windows99_vmx_source_backup_search_receipt_readback_present"] is True + assert state["windows99_vmx_source_backup_search_host_count"] == 7 + assert state["windows99_vmx_source_backup_search_searched_host_count"] == 4 + assert state["windows99_vmx_source_backup_search_blocked_host_count"] == 3 + assert state["windows99_vmx_source_backup_search_blocked_host_aliases"] == [ + "110", + "188", + "112", + ] + assert state["windows99_vmx_source_backup_search_candidate_source_count"] == 0 + assert state["windows99_vmx_source_backup_search_candidate_source_found"] is False + assert ( + state["windows99_vmx_source_backup_search_apply_allowed_by_this_package"] + is False + ) + assert state["windows99_vmx_source_backup_search_raw_path_output"] is False + assert state["windows99_vmx_source_backup_search_path_fingerprint_only"] is True + assert state["windows99_vmx_source_backup_search_file_content_read"] is False + assert state["windows99_vmx_source_backup_search_remote_write_performed"] is False + assert ( + state["windows99_vmx_source_backup_search_vm_power_change_performed"] is False + ) assert payload["summary"]["controlled_service_data_backup_blocker_count"] == 0 assert payload["summary"]["active_p0_primary_blocker"] == ( "reboot_event_required_host_unreachable" @@ -992,28 +1014,28 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a "111" ] assert payload["summary"]["windows99_vmx_source_locator_status"] == ( - "collector_readback_verified_source_required_before_relink_dry_run" + "collector_readback_no_scoped_relink_candidate" ) assert payload["summary"]["windows99_vmx_source_locator_check_mode_ready"] is True assert payload["summary"]["windows99_vmx_source_locator_target_aliases"] == [ "111" ] - assert payload["summary"]["windows99_vmx_source_locator_candidate_source_count"] == 6 + assert payload["summary"]["windows99_vmx_source_locator_candidate_source_count"] == 0 assert ( payload["summary"][ "windows99_vmx_source_locator_identity_unassigned_ubuntu_candidate_count" ] - == 1 + == 0 ) assert ( payload["summary"][ "windows99_vmx_source_locator_candidate_confirmation_required" ] - is True + is False ) assert payload["summary"][ "windows99_vmx_source_locator_candidate_confirmation_status" - ] == "blocked_single_unassigned_ubuntu_candidate_identity_not_confirmed" + ] == "blocked_no_candidate_source_found" assert payload["summary"][ "windows99_vmx_source_locator_candidate_identity_decision" ] == "blocked" @@ -1045,7 +1067,7 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a payload["summary"][ "windows99_vmx_source_locator_verified_source_or_identity_evidence_required" ] - is True + is False ) assert ( payload["rollups"][ @@ -1055,15 +1077,20 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a ) assert payload["rollups"][ "windows99_vmx_source_locator_controlled_relink_dry_run_blocker" - ] == "verified_vmx_source_or_identity_evidence_missing" + ] == "scoped_relink_candidate_not_verified" assert ( payload["rollups"]["windows99_vmx_source_locator_remote_write_performed"] is False ) assert ( payload["rollups"]["windows99_vmx_source_locator_file_content_read"] - is True + is False ) + assert payload["summary"]["windows99_vmx_source_backup_search_status"] == ( + "blocked_windows99_vmx_backup_source_not_found_some_hosts_unsearched" + ) + assert payload["summary"]["windows99_vmx_source_backup_search_candidate_source_count"] == 0 + assert payload["rollups"]["windows99_vmx_source_backup_search_blocked_host_count"] == 3 assert "service/data/backup readback is green" in payload[ "next_execution_order" ][0] @@ -1114,7 +1141,7 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a "windows99_vmx_source_repair_playbook_trust_writeback_ref" ].startswith("playbook://awoooi/windows99-vmx-source-repair/") assert in_progress["evidence"]["windows99_vmx_source_locator_status"] == ( - "collector_readback_verified_source_required_before_relink_dry_run" + "collector_readback_no_scoped_relink_candidate" ) assert ( in_progress["evidence"]["windows99_vmx_source_locator_check_mode_ready"] @@ -1126,17 +1153,17 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a ) assert ( in_progress["evidence"]["windows99_vmx_source_locator_candidate_source_count"] - == 6 + == 0 ) assert ( in_progress["evidence"][ "windows99_vmx_source_locator_candidate_confirmation_required" ] - is True + is False ) assert in_progress["evidence"][ "windows99_vmx_source_locator_candidate_confirmation_status" - ] == "blocked_single_unassigned_ubuntu_candidate_identity_not_confirmed" + ] == "blocked_no_candidate_source_found" assert ( in_progress["evidence"][ "windows99_vmx_source_locator_candidate_relink_dry_run_ready" @@ -1159,7 +1186,7 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a in_progress["evidence"][ "windows99_vmx_source_locator_verified_source_or_identity_evidence_required" ] - is True + is False ) assert ( in_progress["evidence"][ @@ -1171,6 +1198,20 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a in_progress["evidence"]["windows99_vmx_source_locator_remote_write_performed"] is False ) + assert in_progress["evidence"]["windows99_vmx_source_backup_search_package_status"] == ( + "blocked_windows99_vmx_backup_source_not_found_some_hosts_unsearched" + ) + assert ( + in_progress["evidence"]["windows99_vmx_source_backup_search_candidate_source_count"] + == 0 + ) + assert in_progress["evidence"]["windows99_vmx_source_backup_search_blocked_host_count"] == 3 + assert ( + in_progress["evidence"][ + "windows99_vmx_source_backup_search_apply_allowed_by_this_package" + ] + is False + ) assert "Windows99 verify collection" in in_progress["professional_fix"][ "action" ] diff --git a/apps/api/tests/test_windows99_vmx_source_backup_search_package_api.py b/apps/api/tests/test_windows99_vmx_source_backup_search_package_api.py new file mode 100644 index 000000000..a0fc3e4a3 --- /dev/null +++ b/apps/api/tests/test_windows99_vmx_source_backup_search_package_api.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.api.v1 import agents +from src.api.v1.agents import router +from src.services.windows99_vmx_source_backup_search_package import ( + build_windows99_vmx_source_backup_search_package, +) +from src.services.windows99_vmx_source_locator_readback import ( + build_windows99_vmx_source_locator_readback, +) +from src.services.windows99_vmx_source_repair_package import ( + build_windows99_vmx_source_repair_package, +) + + +def _scorecard() -> dict: + return { + "status": "blocked_reboot_auto_recovery_slo_not_ready", + "generated_at": "2026-07-03T17:57:16+08:00", + "windows99_vmware_autostart": { + "required_vm_aliases": ["111", "188", "120", "121", "112"], + "missing_vmx_aliases": ["111"], + "powered_off_aliases": ["111", "112", "120", "121", "188"], + }, + "readback": { + "windows99_no_secret_collector_status": ( + "collected_windows99_vmware_verify_stdout" + ), + "windows99_no_secret_collector_readback_present": True, + "windows99_no_secret_collector_ssh_batchmode_auth_ready": True, + "windows99_no_secret_collector_remote_verify_attempted": True, + }, + "windows99_management_channel": { + "remote_execution_channel_ready": False, + "can_collect_vmware_verify_without_secret": False, + "winrm_http_open": False, + "winrm_https_open": False, + "local_console_channel_reachable": True, + "console_collection_channels": ["rdp_console", "hyperv_vmconnect"], + "ssh_batch": { + "checked": True, + "ready": False, + "status": "permission_denied", + }, + }, + } + + +def _locator_readback() -> dict: + repair_package = build_windows99_vmx_source_repair_package(_scorecard()) + return build_windows99_vmx_source_locator_readback( + _scorecard(), + repair_package=repair_package, + collector_receipt={ + "schema_version": "windows99_vmx_source_locator_receipt_v1", + "collector_readback_present": True, + "collector_collection_status": ( + "collected_windows99_vmx_source_locator_stdout" + ), + "target_host_alias": "99", + "target_vm_aliases": ["111"], + "expected_vmx_path_present_count": 0, + "candidate_source_count": 0, + "candidate_source_fingerprint_count": 0, + "alias_hint_candidate_count": 0, + "unassigned_ubuntu_candidate_count": 0, + "identity_metadata_read": False, + "bounded_identity_metadata_only": False, + "identity_target_hint_count": 0, + "identity_unassigned_ubuntu_candidate_count": 0, + "locate_status": "blocked_missing_expected_vmx_source_no_candidate_found", + }, + ) + + +def _backup_search_receipt() -> dict: + return { + "schema_version": "windows99_vmx_source_backup_search_receipt_v1", + "collector_readback_present": True, + "target_host_alias": "99", + "target_vm_aliases": ["111"], + "host_rows": [ + { + "host_alias": "99", + "status": "not_matched", + "candidate_source_count": 0, + "candidate_source_count_known": True, + "candidate_source_fingerprint_count": 0, + }, + { + "host_alias": "120", + "status": "searched", + "candidate_source_count": 0, + "candidate_source_count_known": True, + "candidate_source_fingerprint_count": 0, + }, + { + "host_alias": "121", + "status": "searched", + "candidate_source_count": 0, + "candidate_source_count_known": True, + "candidate_source_fingerprint_count": 0, + }, + { + "host_alias": "188", + "status": "blocked_ssh_batchmode_unavailable", + "candidate_source_count": 0, + "candidate_source_count_known": False, + "candidate_source_fingerprint_count": 0, + }, + ], + } + + +def test_windows99_vmx_source_backup_search_package_builder_is_check_mode_only() -> None: + payload = build_windows99_vmx_source_backup_search_package( + _scorecard(), + locator_readback=_locator_readback(), + collector_receipt=_backup_search_receipt(), + generated_at="2026-07-03T17:58:00+08:00", + ) + + assert payload["schema_version"] == "windows99_vmx_source_backup_search_package_v1" + assert payload["status"] == ( + "blocked_windows99_vmx_backup_source_not_found_some_hosts_unsearched" + ) + assert payload["package_ready"] is True + assert payload["target_vm_aliases"] == ["111"] + assert payload["source_locator_candidate_source_count"] == 0 + assert payload["receipt_readback_present"] is True + assert payload["search_host_count"] == 4 + assert payload["searched_host_count"] == 3 + assert payload["blocked_host_count"] == 1 + assert payload["blocked_host_aliases"] == ["188"] + assert payload["candidate_source_count"] == 0 + assert payload["candidate_source_found"] is False + assert payload["next_executable_step"] == ( + "rerun_backup_search_for_blocked_hosts_or_offline_inventory" + ) + assert payload["apply_allowed_by_this_package"] is False + assert payload["raw_path_output"] is False + assert payload["path_fingerprint_only"] is True + assert payload["file_content_read"] is False + assert payload["secret_value_read"] is False + assert payload["remote_write_performed"] is False + assert payload["vm_power_change_performed"] is False + assert payload["vm_power_change_allowed"] is False + assert payload["destructive_restore_allowed"] is False + assert "read_vmx_file_contents" in payload["forbidden_actions"] + assert payload["operation_boundaries"]["path_fingerprint_only"] is True + assert payload["post_search_verifier"] == [ + "GET /api/v1/agents/windows99-vmx-source-backup-search-package", + "GET /api/v1/agents/windows99-vmx-source-locator-readback", + "GET /api/v1/agents/reboot-auto-recovery-slo-scorecard", + "GET /api/v1/agents/awoooi-priority-work-order-readback", + ] + + +def test_windows99_vmx_source_backup_search_endpoint_is_public_safe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + agents, + "load_latest_windows99_vmx_source_backup_search_package", + lambda: build_windows99_vmx_source_backup_search_package( + _scorecard(), + locator_readback=_locator_readback(), + collector_receipt=_backup_search_receipt(), + ), + ) + app = FastAPI() + app.include_router(router, prefix="/api/v1") + client = TestClient(app) + + response = client.get("/api/v1/agents/windows99-vmx-source-backup-search-package") + + assert response.status_code == 200 + payload = response.json() + assert payload["schema_version"] == "windows99_vmx_source_backup_search_package_v1" + assert payload["target_vm_aliases"] == ["111"] + assert payload["candidate_source_count"] == 0 + assert payload["blocked_host_aliases"] == ["188"] + assert payload["public_lan_topology_redacted"] is True + assert payload["raw_path_output"] is False + assert payload["file_content_read"] is False + assert payload["remote_write_performed"] is False + assert payload["vm_power_change_performed"] is False + assert "192.168.0." not in response.text + assert "D:\\" not in response.text + assert "/Users/" not in response.text + assert "/Volumes/" not in response.text diff --git a/apps/api/tests/test_windows99_vmx_source_locator_readback_api.py b/apps/api/tests/test_windows99_vmx_source_locator_readback_api.py index 998f8f660..feeb19e87 100644 --- a/apps/api/tests/test_windows99_vmx_source_locator_readback_api.py +++ b/apps/api/tests/test_windows99_vmx_source_locator_readback_api.py @@ -219,32 +219,33 @@ def test_windows99_vmx_source_locator_endpoint_returns_public_safe_readback( payload = response.json() assert payload["schema_version"] == "windows99_vmx_source_locator_readback_v1" assert payload["status"] == ( - "collector_readback_verified_source_required_before_relink_dry_run" + "collector_readback_no_scoped_relink_candidate" ) assert payload["target_vm_aliases"] == ["111"] assert payload["locator_check_mode_ready"] is True assert payload["effective_locator_collect_ready"] is False assert payload["locator_collect_blocker"] == "ssh_batch_permission_denied" assert payload["collector_readback_present"] is True - assert payload["candidate_source_count"] == 6 - assert payload["candidate_confirmation_required"] is True + assert payload["candidate_source_count"] == 0 + assert payload["candidate_confirmation_required"] is False assert payload["human_confirmation_required_for_dry_run"] is False - assert payload["verified_source_or_identity_evidence_required"] is True + assert payload["verified_source_or_identity_evidence_required"] is False assert payload["controlled_relink_dry_run_ready"] is False assert payload["controlled_relink_dry_run_blocker"] == ( - "verified_vmx_source_or_identity_evidence_missing" + "scoped_relink_candidate_not_verified" ) assert payload["controlled_relink_dry_run_plan"]["plan_id"] == ( "windows99-vmx-source-relink-dry-run-111" ) assert payload["controlled_relink_dry_run_plan"]["remote_write_performed"] is False + assert payload["candidate_confirmation_status"] == "blocked_no_candidate_source_found" assert payload["candidate_identity_evidence_sufficient"] is False assert payload["candidate_relink_dry_run_ready"] is False assert payload["apply_allowed_by_candidate_confirmation_gate"] is False assert payload["public_lan_topology_redacted"] is True assert payload["raw_path_output"] is False - assert payload["file_content_read"] is True - assert payload["bounded_identity_metadata_only"] is True + assert payload["file_content_read"] is False + assert payload["bounded_identity_metadata_only"] is False assert payload["remote_write_performed"] is False assert payload["vm_power_change_performed"] is False assert "192.168.0." not in response.text diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 3ac613eb3..19c2cf70c 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,24 @@ +## 2026-07-03 — 17:57 Windows99 VMX backup/source search package + +**完成內容**： +- P0-4 繼續收斂 111 VMX source blocker；最新 via-110 no-secret locator 已讀回 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`，並取代早先 `candidate_source_count=6` / generic Ubuntu candidate 的歷史 receipt。 +- 新增 `scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh`：以 SSH public-key / BatchMode 搜尋 Linux backup roots，只輸出 path fingerprints 與 count，不讀 VMX 內容、不輸出 raw path、不寫遠端、不啟停 VM、不重啟服務或主機。 +- 新增 `/api/v1/agents/windows99-vmx-source-backup-search-package`、`windows99_vmx_source_backup_search_package` service 與 priority readback 欄位：目前 receipt 顯示 99 locator / 120 / 121 / local 未找到 verified 111 source，110 transport intermittent，188 / 112 SSH BatchMode unavailable；`candidate_source_count=0`、`blocked_host_count=3`、`apply_allowed_by_this_package=false`。 +- 更新 `docs/operations/windows99-vmx-source-locator-receipt.snapshot.json` 與新增 `docs/operations/windows99-vmx-source-backup-search-receipt.snapshot.json`，讓 production API 可讀回最新 no-candidate / backup-search truth。 +- `FULL-STACK-COLD-START-SOP` 更新至 v1.129，`REBOOT-RECOVERY-SOP` 更新至 v5.20，統帥插入需求台帳更新 P0-4 當前狀態，避免再用舊 generic candidate 當目前 blocker。 + +**已跑驗證**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest scripts/reboot-recovery/tests/test_windows99_vmx_source_backup_search.py apps/api/tests/test_windows99_vmx_source_backup_search_package_api.py apps/api/tests/test_windows99_vmx_source_locator_readback_api.py -q -p no:cacheprovider`：`7 passed`。 +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_awoooi_priority_work_order_readback_api.py -q -p no:cacheprovider`：`19 passed`。 +- `python3.11 -m py_compile apps/api/src/services/windows99_vmx_source_backup_search_package.py apps/api/src/api/v1/agents.py apps/api/src/services/awoooi_priority_work_order_readback.py`、`bash -n scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh`：通過。 + +**仍維持**： +- 這不代表 111 VMX source 已恢復；目前真 blocker 仍是 verified VMX source / backup path 缺失、guest powered off、Windows99 autostart config not ready / task never ran，以及 primary `reboot_event_required_host_unreachable`。 +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有使用 GitHub / gh；沒有啟動或關閉 VM，沒有 relink / restore，沒有重啟 Windows host / VMware service，沒有 Docker / Nginx / K3s / DB / firewall restart / reload，沒有 prune / delete / force push。 + +**下一步**： +- 跑 full focused verifier / runner pressure guard / `git diff --check` 後 commit / push Gitea main；CD 後讀回 production backup-search endpoint、priority readback 與 SLO scorecard。P0-4 下一個可執行動作是 rerun blocked hosts 的 backup/source search 或離線備份 inventory，找到 verified source path 後才建立 scoped relink dry-run。 + ## 2026-07-03 — 17:34 Windows99 no-secret verifier timeout 調整 **完成內容**： diff --git a/docs/operations/windows99-vmx-source-backup-search-receipt.snapshot.json b/docs/operations/windows99-vmx-source-backup-search-receipt.snapshot.json new file mode 100644 index 000000000..bf7de08f3 --- /dev/null +++ b/docs/operations/windows99-vmx-source-backup-search-receipt.snapshot.json @@ -0,0 +1,104 @@ +{ + "schema_version": "windows99_vmx_source_backup_search_receipt_v1", + "generated_at": "2026-07-03T17:57:16+08:00", + "receipt_ref": "log://awoooi/p0-006/windows99-vmx-source-backup-search-111-20260703-1757", + "source": "live_no_secret_filename_search", + "collector_readback_present": true, + "collector_schema_version": "windows99_vmx_source_backup_search_collector_v1", + "target_host_alias": "99", + "target_vm_aliases": ["111"], + "search_mode": "filename_path_fingerprint_only", + "host_rows": [ + { + "host_alias": "99", + "search_role": "windows_vmware_roots_via_locator", + "status": "not_matched", + "candidate_source_count": 0, + "candidate_source_count_known": true, + "candidate_source_fingerprint_count": 0, + "collector": "windows99-vmx-source-locator-readback" + }, + { + "host_alias": "110", + "search_role": "linux_backup_roots", + "status": "transport_intermittent", + "candidate_source_count": 0, + "candidate_source_count_known": false, + "candidate_source_fingerprint_count": 0, + "collector": "collect-windows99-vmx-source-backup-search.sh" + }, + { + "host_alias": "120", + "search_role": "linux_backup_roots", + "status": "searched", + "candidate_source_count": 0, + "candidate_source_count_known": true, + "candidate_source_fingerprint_count": 0, + "collector": "collect-windows99-vmx-source-backup-search.sh" + }, + { + "host_alias": "121", + "search_role": "linux_backup_roots", + "status": "searched", + "candidate_source_count": 0, + "candidate_source_count_known": true, + "candidate_source_fingerprint_count": 0, + "collector": "collect-windows99-vmx-source-backup-search.sh" + }, + { + "host_alias": "188", + "search_role": "linux_backup_roots", + "status": "blocked_ssh_batchmode_unavailable", + "candidate_source_count": 0, + "candidate_source_count_known": false, + "candidate_source_fingerprint_count": 0, + "collector": "collect-windows99-vmx-source-backup-search.sh" + }, + { + "host_alias": "112", + "search_role": "linux_backup_roots", + "status": "blocked_ssh_batchmode_unavailable", + "candidate_source_count": 0, + "candidate_source_count_known": false, + "candidate_source_fingerprint_count": 0, + "collector": "collect-windows99-vmx-source-backup-search.sh" + }, + { + "host_alias": "local", + "search_role": "operator_workstation_filename_search", + "status": "not_matched", + "candidate_source_count": 0, + "candidate_source_count_known": true, + "candidate_source_fingerprint_count": 0, + "non_target_vmx_observed_count": 1, + "collector": "bounded_local_filename_search" + } + ], + "searched_host_count": 4, + "blocked_host_count": 3, + "candidate_source_count": 0, + "candidate_source_fingerprint_count": 0, + "search_status": "blocked_no_verified_vmx_source_found_some_hosts_unsearched", + "safe_next_step": "rerun_windows99_vmx_source_backup_search_check_mode_for_blocked_hosts_then_update_receipt_no_write", + "public_lan_topology_redacted": true, + "redacted_public_display_only": true, + "raw_path_output": false, + "path_fingerprint_only": true, + "file_content_read": false, + "secret_value_read": false, + "password_prompt_allowed": false, + "remote_write_performed": false, + "host_reboot_performed": false, + "vm_power_change_performed": false, + "windows_service_restart_performed": false, + "windows_registry_apply_performed": false, + "scheduled_task_write_performed": false, + "km_writeback_ref": "km://awoooi/reboot-slo/windows99-vmx-source-backup-search-111", + "playbook_trust_writeback_ref": "playbook://awoooi/windows99-vmx-source-backup-search/windows99-vmx-source-backup-search-111", + "post_search_verifier": [ + "GET /api/v1/agents/windows99-vmx-source-backup-search-package", + "GET /api/v1/agents/windows99-vmx-source-locator-readback", + "GET /api/v1/agents/reboot-auto-recovery-slo-scorecard", + "GET /api/v1/agents/awoooi-priority-work-order-readback" + ] +} diff --git a/docs/operations/windows99-vmx-source-locator-receipt.snapshot.json b/docs/operations/windows99-vmx-source-locator-receipt.snapshot.json index b09de0313..e2318bcc0 100644 --- a/docs/operations/windows99-vmx-source-locator-receipt.snapshot.json +++ b/docs/operations/windows99-vmx-source-locator-receipt.snapshot.json @@ -1,9 +1,9 @@ { "schema_version": "windows99_vmx_source_locator_receipt_v1", - "generated_at": "2026-07-03T12:41:34+08:00", - "receipt_ref": "log://awoooi/p0-006/windows99-vmx-source-locator-111", + "generated_at": "2026-07-03T17:57:16+08:00", + "receipt_ref": "log://awoooi/p0-006/windows99-vmx-source-locator-111-20260703-1757", "source": "live_via_110_no_secret_check_mode", - "collector_schema_version": "windows99_vmx_source_locate_collector_v1", + "collector_schema_version": "windows99_vmx_source_locator_collector_v1", "collector_readback_present": true, "collector_collection_status": "collected_windows99_vmx_source_locator_stdout", "target_host_alias": "99", @@ -11,18 +11,20 @@ "target_vm_aliases": ["111"], "expected_vmx_path_count": 1, "expected_vmx_path_present_count": 0, - "candidate_source_count": 6, + "discovery_root_count": 6, + "discovery_root_present_count": 2, + "candidate_source_count": 0, "candidate_source_count_known": true, "candidate_source_fingerprint_count": 0, "alias_hint_candidate_count": 0, - "unassigned_ubuntu_candidate_count": 5, - "identity_metadata_read": true, - "bounded_identity_metadata_only": true, + "unassigned_ubuntu_candidate_count": 0, + "identity_metadata_read": false, + "bounded_identity_metadata_only": false, "identity_target_hint_count": 0, - "identity_unassigned_ubuntu_candidate_count": 1, - "locate_status": "single_identity_unassigned_ubuntu_candidate_requires_confirmation", - "safe_next_step": "confirm_single_unassigned_ubuntu_candidate_is_target_alias_before_scoped_relink_dry_run", - "candidate_confirmation_required": true, + "identity_unassigned_ubuntu_candidate_count": 0, + "locate_status": "blocked_missing_expected_vmx_source_no_candidate_found", + "safe_next_step": "provide_existing_vmx_source_or_backup_path_then_rerun_locator_check_mode_no_write", + "candidate_confirmation_required": false, "controlled_relink_dry_run_ready": false, "apply_allowed_by_this_receipt": false, "public_lan_topology_redacted": true, @@ -46,6 +48,7 @@ "bash scripts/reboot-recovery/collect-windows99-vmware-verify.sh --collect", "bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh --verify-only", "GET /api/v1/agents/windows99-vmx-source-locator-readback", + "GET /api/v1/agents/windows99-vmx-source-backup-search-package", "GET /api/v1/agents/reboot-auto-recovery-slo-scorecard", "GET /api/v1/agents/awoooi-priority-work-order-readback" ] diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 20e6f007b..91ddad352 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.128 +> Version: v1.129 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -10,6 +10,8 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天，必須先重跑 live check，再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 +v1.129 Windows99 VMX backup/source search package rule：2026-07-03 17:57 最新 via-110 no-secret locator 已把目前 truth 從早先 `candidate_source_count=6` 修正為 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`，且 `raw_path_output=false`、`path_fingerprint_only=true`、`secret_value_read=false`、`remote_write_performed=false`、`vm_power_change_performed=false`。因此 P0-4 下一步不得再引用 generic Ubuntu candidate；必須讀 `/api/v1/agents/windows99-vmx-source-backup-search-package` 與 priority readback 的 `windows99_vmx_source_backup_search_*` 欄位。當前 backup/source search receipt 顯示 99/120/121/local 均未找到 verified 111 source，110 為 transport intermittent，188/112 SSH BatchMode 不可用，`candidate_source_count=0`、`blocked_host_count=3`、`apply_allowed_by_this_package=false`。固定下一步是 rerun backup search for blocked hosts 或離線備份 inventory，再取得 verified source path 後建立 scoped relink dry-run；仍不得 power on VM、restore、讀 VMX 內容、讀 secret、重啟 Windows / VMware service / host / Docker / Nginx / K3s / DB / firewall。 + 若只是重啟後要快速判斷能不能宣稱恢復，先跑機器可讀摘要：`scripts/reboot-recovery/post-reboot-readiness-summary.sh --no-color`。此腳本會呼叫一頁式總檢查、188 host hygiene checklist 與 Wazuh no-false-green repo gates，並把 delegated logs 和可重放的 `summary.txt` 留在 `/tmp/awoooi-post-reboot-readiness-*`。v1.75 起，同一輪驗收後續步驟必須吃同一個 `$ARTIFACT_DIR/summary.txt`，例如 `scripts/reboot-recovery/post-reboot-declaration-guard.py --summary-file "$ARTIFACT_DIR/summary.txt" --no-color` 與 `scripts/reboot-recovery/post-reboot-next-gate-dispatch.sh --summary-file "$ARTIFACT_DIR/summary.txt" --no-color`；不得在同一輪 evidence chain 反覆重跑 live probes 後混用不同時間點結論。v1.76 起，delegated cold-start 若在 K3s rollout / CD 替換瞬間出現單次 `BLOCKED AWOOOI API not reachable`，但 wrapper 自己的 public `https://awoooi.wooo.work/api/v1/health` route retry 已回 2xx，該 blocker 只列為 route/API warmup evidence warning；public API 仍失敗、其他 non-route blocker、或 retry 後未恢復時，仍維持 hard blocked。宣告 guard 會把 summary 轉成 allowed / forbidden declaration，避免把服務綠誤報成 DR complete、188 host hygiene、Wazuh registry recovered 或 runtime authorized。若 summary 顯示 `SERVICE_GREEN=1` 但 `NEXT_REQUIRED_GATES` 仍非空，再由 dispatch checklist 把尚未完成的 blocker 轉成 owner / evidence / forbidden-action checklist；需要機器可讀 intake 時，再跑 `scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py --summary-file "$ARTIFACT_DIR/summary.txt" --output /tmp/awoooi-post-reboot-owner-packets.json` 產生 `awoooi_post_reboot_next_gate_owner_packets_v1` JSON，立刻跑 `scripts/reboot-recovery/post-reboot-owner-packet-contract-guard.py --packet-file /tmp/awoooi-post-reboot-owner-packets.json`，並用 `scripts/reboot-recovery/post-reboot-owner-response-template.py --owner-packet-file /tmp/awoooi-post-reboot-owner-packets.json --output /tmp/awoooi-post-reboot-owner-response-template.json` 產生只含 active gates 的 placeholder response。dispatch / packet / guard / template 均固定 `DISPATCH_AUTHORIZED=0`、`REQUEST_SENT_COUNT=0`、`OWNER_RESPONSE_ACCEPTED=0`、`HOST_WRITE_AUTHORIZED=0`、`SECRET_VALUE_COLLECTION_ALLOWED=0`、`RUNTIME_GATE=0`；guard 未通過時不得送 owner request、不得寫 escrow marker、不得進維護窗口、不得宣稱 DR / Wazuh registry complete。v1.74 起，任何 owner response JSON 還必須經過 `scripts/reboot-recovery/post-reboot-owner-response-preflight.py --no-color --owner-packet-file --response-file `：空模板、placeholder、secret payload、runtime action request、credential marker write、Wazuh active response / re-enroll / restart、Kali active scan 或缺少 active gate evidence 都必須 fail-closed；preflight 通過也只表示可進入獨立 reviewer acceptance，不是 runtime 授權。需要人工展開時，再跑 `scripts/reboot-recovery/post-start-quick-check.sh --no-color` 並以 `docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md` 作為 fallback。長 SOP 保留完整背景、例外處理與 Plan B；短版 wrapper / checklist 負責每次 T+10 分鐘內的固定判定。 v1.76 owner gate replay rule：同一輪 summary 產生後，owner packet 與 owner response preflight 必須優先使用 `--summary-file "$ARTIFACT_DIR/summary.txt"`，例如 `scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py --no-color --summary-file "$ARTIFACT_DIR/summary.txt" --output /tmp/awoooi-post-reboot-owner-packets.json` 與 `scripts/reboot-recovery/post-reboot-owner-response-preflight.py --no-color --summary-file "$ARTIFACT_DIR/summary.txt" --response-file `。只有在刻意要重新取 live evidence 時，才允許省略 `--summary-file`；否則 preflight 不得自己重跑 summary 造成同一輪狀態漂移。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 957881358..ae7dce236 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.19 +> **版本**: v5.20 > **最後更新**: 2026-07-03 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-03 17:34 Windows99 no-secret verifier timeout 預算修正，避免 90 秒假 timeout 遮蔽 111 VMX source blocker +> **觸發事件**: 2026-07-03 17:57 Windows99 VMX locator no-candidate truth 與 backup/source search package 上卷 --- @@ -21,6 +21,14 @@ ## 架構概覽與依賴圖 +### 2026-07-03 17:57 P0-4 VMX Backup / Source Search 覆蓋 + +最新 via-110 no-secret locator 已成功收斂為 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`，早先 `candidate_source_count=6` / generic Ubuntu candidate 僅保留為歷史 receipt，不再作為目前 relink 候選。`docs/operations/windows99-vmx-source-locator-receipt.snapshot.json` 已更新為 17:57 no-candidate truth。 + +新增 `/api/v1/agents/windows99-vmx-source-backup-search-package` 與 `scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh`。此 package / collector 固定只做 filename / path-fingerprint 搜尋，不讀 VMX 內容、不輸出 raw path、不讀 secret、不寫遠端、不改 Windows、不啟停 VM、不重啟任何服務或主機。當前 receipt：99 locator / 120 / 121 / local 均未找到 verified 111 source，110 為 transport intermittent，188 / 112 SSH BatchMode 不可用；`candidate_source_count=0`、`blocked_host_count=3`、`apply_allowed_by_this_package=false`。 + +P0-4 下一步固定為 rerun blocked hosts 的 backup/source search 或離線備份 inventory，取得 verified source path 後才可建立 scoped relink dry-run + rollback + post-verifier；不得猜 generic Ubuntu candidate、不得直接 relink、不得 restore、不得 power on VM、不得讀 Windows 密碼或 VMX 內容。 + ### 2026-07-03 17:34 P0-4 Verify Timeout 覆蓋 110→99 SSH public-key channel 可用，via-110 locator 已成功收集且未讀 secret / 未寫 Windows / 未改 VM power：`expected_vmx_path_present_count=0`、`discovery_root_present_count=2`、`candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`。同輪 full VMware Verify 在 90 秒曾 timeout，但 180 秒成功讀回 `MISSING_VMX_ALIASES=111`、`VMWARE_AUTOSTART_CONFIG_READY=0`、`VMWARE_AUTOSTART_VERIFY_READY=0`。 diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 1895db5cf..6eee6b533 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -57,14 +57,14 @@ | 順序 | ID | 優先序 | 使用者插入要求 | 正規化工作項 | 目前狀態 | 下一個可驗證動作 | | --- | --- | --- | --- | --- | --- | --- | -| 1 | CIR-P0-RBT-001 | P0 | 「主機重啟後 10 分鐘內全部恢復，且要自動判斷所有主機被重啟」 | 建立 99/110/111/112/120/121/188 reboot event detector + 10 分鐘 SLO scorecard + fixed triage order | 2026-07-03 12:34 deploy 後 production scorecard：readiness `60%`、active blockers `10`、primary `reboot_event_required_host_unreachable`、`can_claim_all_services_recovered_within_target=false`；locator endpoint 已 ready，missing alias `111`，但 VMX source / guest power / autostart config / Windows Update policy / reboot event 仍 blocked | 優先收斂 111 VMX source locate / verified backup artifact、111 reachability 與 fresh all-host reboot event；不可宣稱 10 分鐘 SLA 已證明，不得用 RDP clipboard 片段或 generic VMX guess 當完成證據 | +| 1 | CIR-P0-RBT-001 | P0 | 「主機重啟後 10 分鐘內全部恢復，且要自動判斷所有主機被重啟」 | 建立 99/110/111/112/120/121/188 reboot event detector + 10 分鐘 SLO scorecard + fixed triage order | 2026-07-03 17:57 最新 truth：production/runtime 仍 `can_claim_all_services_recovered_within_target=false`，primary `reboot_event_required_host_unreachable`；Windows99 locator 已從舊 generic candidate 收斂為 `candidate_source_count=0`，backup/source search package 顯示 99/120/121/local 未找到 verified 111 source、110 transport intermittent、188/112 SSH BatchMode unavailable | 優先收斂 111 verified VMX source / backup path、blocked hosts backup search、111 reachability 與 fresh all-host reboot event；不可宣稱 10 分鐘 SLA 已證明，不得用 RDP clipboard 片段或 generic VMX guess 當完成證據 | | 2 | CIR-P0-RBT-002 | P0 | 「沒有偵測到主機重啟」 | 修正 host reboot/shutdown/up detection：boot_id / uptime / node exporter / Windows exporter / VMware VM power state 都要進同一事件 | Scorecard 已接 collection packet + management probe；08:23 host probe 有 7 列但 99 只有 ping/RDP reachable、uptime unknown，111 unreachable，112/120/121/188 uptime 都已超過 10 分鐘窗口 | 讓 99 verifier / Windows exporter 或等效 no-secret readback 進入 host boot event，並補 111 reachability 證據 | -| 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；12:00 API / priority readback 顯示 `source_repair_check_mode_ready`、下一步 `locate_or_relink_missing_vmx_source_check_mode`；本輪 no-write locator 確認 expected 111 VMX path `exists=0`、`candidate_vmx_count=6`、`alias_hint_candidate_count=0`、`identity_target_hint_count=0`、`identity_unassigned_ubuntu_candidate_count=1`、`locate_status=single_identity_unassigned_ubuntu_candidate_requires_confirmation`；VMware services / scheduled task / Windows Update policy 已 ok，但 guest power 仍 not ready | 不得直接 relink generic Ubuntu candidate；先取得 external verified backup artifact 或更強 identity evidence，再做 scoped relink / restore dry-run 與 post-verifier；不得從 public redacted path 或 scorecard 直接啟動 / 關閉 VM、重啟 Windows service、讀密碼或 host reboot | +| 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；17:57 no-write locator 確認 expected 111 VMX path 不存在、`candidate_source_count=0`、`locate_status=blocked_missing_expected_vmx_source_no_candidate_found`；backup/source package 顯示 `candidate_source_count=0`、`blocked_host_count=3`、`apply_allowed_by_this_package=false`；VMware services / scheduled task / Windows Update policy readback 已有，但 guest power 仍 not ready | 不得直接 relink generic Ubuntu candidate；先 rerun blocked hosts backup/source search 或離線備份 inventory，取得 verified source path 後再做 scoped relink dry-run、rollback 與 post-verifier；不得從 public redacted path 或 scorecard 直接啟動 / 關閉 VM、重啟 Windows service、讀密碼或 host reboot | | 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | 10:15 source/API patch：若 nested verifier 回 `policy_blockers=["windows_update_policy_readback_missing"]`，API 必須 fail-closed 重新提升 `windows99_update_no_auto_reboot_policy_not_ready`；不得只信舊 Prometheus `windows99_update_no_auto_reboot_ready=true` | 補 no-secret policy readback / drift alert；若 policy 不綠，另走 controlled apply package + rollback + post-verifier，禁止要求或記錄 Windows 密碼，禁止由 scorecard 直接 apply registry | | 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；Gitea CD `#4459` Success、deploy marker `8d7a6faaf`，production API scorecard 已讀回 `public_maintenance_fallback.ready=true`、raw 5xx=`0`、unreachable without L1=`0`，P0 blockers `11`、readiness `47`；仍不可宣稱整體 10 分鐘 SLO 完成 | 若未來 public route 出現 raw 5xx，先走 L0 Nginx intercept / header verifier；若 edge unreachable，才產生 L1 external static origin / CDN decision record + rollback | | 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | HostDown / HostRebootEventDetected / RebootAutoRecoverySLOMissed 已存在；per-blocker reboot alerts 與 backup receipt rules 已 deploy/readback。Backup receipt 缺段已從 100 條 stage 噪音收斂為 110 / 188 兩條 host-level pending；仍需完整 shutdown/up E2E receipt | 補 Prometheus / Alertmanager active/resolved 與 outbound receipt；backup alert 先補 `/backup/alert-receipts/*.last_success` 脫敏 marker，不送測試 secret、不重啟主機 | | 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram/AwoooP receipt | 已有 backup health exporter / alert rules / Gitea bundle restore dry-run；2026-07-03 runtime 讀回 110 有 88 個 receipt stage requirement、188 有 12 個，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 聚合 pending 110 / 188 各一條 | 補 `/backup/alert-receipts/*.last_success`；再補 Gitea full dump / DB / settings / issues / packages / LFS 與所有工具/log 全量備份監控 | -| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard readback 已固定 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `60%`、active blockers `10`、primary `reboot_event_required_host_unreachable`；12:34 next executable step 固定為 `locate_or_relink_missing_vmx_source_check_mode`，且 locator 結果要求先補 identity / backup evidence | 優先收斂 111 VMX source verified artifact、111 reachability / fresh all-host reboot event、188 startup failed/degraded；不得用不同排查路徑繞過 scorecard | +| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard / priority readback 固定目前 P0：primary `reboot_event_required_host_unreachable`；Windows99 locator 最新 `candidate_source_count=0`，backup/source search package 已把 99/120/121/local searched、110/188/112 blocked/unsearched 轉成機器可讀欄位 | 優先收斂 111 VMX source verified artifact、blocked hosts backup search、111 reachability / fresh all-host reboot event、188 startup failed/degraded；不得用不同排查路徑繞過 scorecard | | 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | AWOOOI Gitea `main=bc8d3af3f chore(cd): deploy dacdd90 [skip ci]`；CD `#4558` log 顯示 API/Web build/push wrapper 全部 `rc=0`、deploy marker `dacdd90`、production deploy readback matched。Production SLO route 已讀回最新 API 行為：`active_blocker_count=10`、`runtime_metric_runtime_readback_added_blockers=["windows99_vmware_autostart_config_not_ready","windows99_update_no_auto_reboot_policy_not_ready"]`、`windows99_update_no_auto_reboot_ready=false`。StockPlatform public freshness / ingestion 仍為 `ok`，latest trading date `2026-07-02`。 | 版本層已補到 AWOOOI production；下一步仍要建立全產品 readback 表，且收斂 Windows99 VMX / service / policy / power 與 host reboot-event blockers | | 10 | CIR-P0-GIT-001 | P0 | 「Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？」 | Gitea repository identity + backup proof + restore drill：不能只看 UI visible，要比對 SSH heads、repo path、bundle backup、restore sample | 2026-07-02 production `/api/v1/agents/gitea-repo-bundle-backup-readback` 已 ready：9 expected repos present/ok、missing=0、failed=0、checksum_missing=0、bundle_fresh=true、all_expected_ok=true、sample_restore_dry_run_ok=true；repo bundle / restore dry-run 層已關閉，不是 repo missing。 | 維持每日 bundle backup + restore dry-run monitoring；另補 Gitea full dump / DB / settings / issues / packages / LFS 備份 readback。禁止刪 repo / 改 visibility / 讀 token / restore 到 production | | 11 | CIR-P0-CPU-001 | P0 | 「110 / 188 CPU 負載持續過高，為什麼沒監控告警、沒主動修復」 | Sustained CPU pressure automation：Alertmanager → controller → evidence → service playbook → verifier → KM writeback | 110 已有 `Host110SustainedModeratePressure`、Gitea playbook、Stock/Postgres evidence；188 仍需同級 controller/alerts readback | 下一步接 `postgres_hot_query_or_backup_export_playbook`；並補 188 equivalent readback，不以單次下降結案 | @@ -134,9 +134,9 @@ | KM / PlayBook / RAG / MCP 整合 | 已被列為 P1，不再遺漏 | 建立 work item schema 與 trust writeback 欄位 | | OpenClaw / Gather-style 持續動畫工作室 | route 已存在，已列為 P1 工作項 | 補 production desktop/mobile smoke、AwoooP 導流與截圖證據 | | AI 專業 UI / 非文字牆 cockpit | 已列為 P2 UX 驗收 | 將長文字區塊收斂成 first-viewport cockpit、cards、flow rows 與 expandable details | -| 10 分鐘 reboot auto-recovery SLA | 2026-07-03 12:34 production scorecard 固定 `10` blockers / readiness `60%`，但 `can_claim_all_services_recovered_within_target=false`；primary `reboot_event_required_host_unreachable`，Windows99 locator endpoint ready，111 VMX source / guest power / config / policy 仍 blocked | 收斂 111 VMX source verified artifact、111 reachability、188 startup failed/degraded，再 rerun host probe + reboot-event detector | +| 10 分鐘 reboot auto-recovery SLA | 2026-07-03 17:57 latest truth：仍 `can_claim_all_services_recovered_within_target=false`；primary `reboot_event_required_host_unreachable`，Windows99 locator `candidate_source_count=0`，backup/source search `candidate_source_count=0` / `blocked_host_count=3`；111 VMX source / guest power / config / policy 仍 blocked | 收斂 111 verified VMX source / backup path、blocked hosts backup search、111 reachability、188 startup failed/degraded，再 rerun host probe + reboot-event detector | | AWOOOI latest source production deploy readback | Gitea `main=bc8d3af3f`，deploy marker `dacdd90`；public CD `#4558` log job succeeded / production deploy readback matched；production scorecard 已讀回 latest API 行為並把 Windows99 config / policy blockers 上卷 | 下一步轉回 Windows99 VMX / service / policy / power 與 host reboot-event blockers；不得因 deploy/readback 成功宣稱 10 分鐘 SLO 完成 | -| 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 已透過 no-secret collector 收集，VMware services / scheduled task / Windows Update policy ok；VMX locate check 顯示 expected 111 path missing、無 alias / identity target hint、只有一個 generic Ubuntu candidate 需再確認 | 取得 external verified backup artifact 或更強 identity evidence，產生 scoped relink / restore dry-run，重跑 no-secret collector，確認 VM 111/188/120/121/112 running | +| 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 已透過 no-secret collector 收集；VMX locator 最新顯示 expected 111 path missing 且 `candidate_source_count=0`，backup/source package 顯示 verified source 未找到、110/188/112 仍需補 search/transport | 取得 verified VMX source / backup path，產生 scoped relink dry-run，重跑 no-secret collector，確認 VM 111/188/120/121/112 running | | Reboot SLO blocker 收斂 | production scorecard 已固定 `10` blockers / readiness `60%`，source / runtime 已補 per-blocker metric/alert；剩餘主 blocker 是 reboot event / unreachable + Windows99 111 VMX source / guest power / config / policy + 111 reachability / 188 degraded | 依具名 blocker 收斂 99/111/188；不得用 route green 或 RDP 可見宣稱 SLA 完成 | | 全備份監控告警 coverage | exporter/rule 已有 host/DB/site/service/package/tool/log coverage 與 backup alert receipt requirement；runtime rules 已部署，缺段 alert 聚合成 110 / 188 host-level pending；production receipt markers / full Gitea dump / DB/settings/issues/packages/LFS 尚未全綠 | 補 `/backup/alert-receipts/*.last_success`、Gitea full dump 與 restore drill verifier | | Stock/Postgres hot pressure | 110 live 已導向 Stock/Postgres playbook；尚未完成 hot query / backup export playbook closure | 下一步執行 read-only Stock/Postgres evidence 與 source freshness / query attribution | @@ -154,7 +154,7 @@ ## 下一輪固定執行順序 1. 先確認 Gitea `main` / CD / production readback 是否仍與最新 truth 一致。 -2. 接續目前 active reboot P0：用 12:34 production scorecard 的固定順序收斂 111 VMX source verified artifact、111 reachability、188 startup failed/degraded，再 rerun host probe / reboot-event detector / scorecard。 +2. 接續目前 active reboot P0：用 17:57 locator / backup-search truth 收斂 111 verified VMX source / backup path、blocked hosts backup search、111 reachability、188 startup failed/degraded，再 rerun host probe / reboot-event detector / scorecard。 3. 補全 reboot auto-recovery P0：99/110/111/112/120/121/188 reboot detection、10 分鐘 SLO scorecard、Telegram down/up/recovered/SLO missed / active-blocker receipts。 4. 補 99 Windows / VMware autostart P0：99 host 自動啟動 VMware，VM 111/188/120/121/112 自動啟動；同時補 Windows Update no-auto-restart verifier。 5. 補 111 reachability / host boot readback，讓 `host_unreachable_after_reboot` 與 `reboot_event_required_host_unreachable` 有單一固定處理路徑。 diff --git a/scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh b/scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh new file mode 100755 index 000000000..6143e6050 --- /dev/null +++ b/scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOSTS="${WINDOWS99_VMX_BACKUP_SEARCH_HOSTS:-110=wooo@192.168.0.110,120=wooo@192.168.0.120,121=wooo@192.168.0.121,188=wooo@192.168.0.188,112=wooo@192.168.0.112}" +SEARCH_ROOTS="${WINDOWS99_VMX_BACKUP_SEARCH_ROOTS:-/home/wooo /mnt /backup /backups /var/backups}" +MAX_DEPTH="${WINDOWS99_VMX_BACKUP_SEARCH_MAX_DEPTH:-7}" +CONNECT_TIMEOUT="${WINDOWS99_VMX_BACKUP_SEARCH_CONNECT_TIMEOUT:-8}" +SSH_TIMEOUT="${WINDOWS99_VMX_BACKUP_SEARCH_TIMEOUT:-35}" +KNOWN_HOSTS_FILE="${WINDOWS99_VMX_BACKUP_SEARCH_KNOWN_HOSTS_FILE:-/tmp/awoooi-windows99-vmx-backup-search-known_hosts}" + +usage() { + cat <<'USAGE' +usage: collect-windows99-vmx-source-backup-search.sh [options] + +Runs a public-key-only, no-content VMX source/backup filename search across +Linux hosts that may hold recovery artifacts for the missing Windows99 111 VMX. +It emits path fingerprints only. It never reads VMX file contents, writes files, +starts/stops VMs, restarts services, changes registry/task state, or reboots. + +Options: + --hosts LIST Comma list like 110=wooo@192.168.0.110,120=wooo@192.168.0.120 + --roots LIST Space-separated roots to search. Default: common backup roots + --max-depth N find maxdepth. Default: 7 + --timeout SECONDS Per-host SSH timeout. Default: 35 + -h, --help Show this help +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --hosts) + shift + HOSTS="${1:?--hosts requires a value}" + ;; + --roots) + shift + SEARCH_ROOTS="${1:?--roots requires a value}" + ;; + --max-depth) + shift + MAX_DEPTH="${1:?--max-depth requires a value}" + ;; + --timeout) + shift + SSH_TIMEOUT="${1:?--timeout requires a value}" + ;; + -h|--help) + usage + exit 0 + ;; + *) + printf 'error=unknown_argument:%s\n' "$1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +SSH_OPTS=( + -o BatchMode=yes + -o PreferredAuthentications=publickey + -o PubkeyAuthentication=yes + -o PasswordAuthentication=no + -o KbdInteractiveAuthentication=no + -o NumberOfPasswordPrompts=0 + -o ConnectTimeout="${CONNECT_TIMEOUT}" + -o ConnectionAttempts=1 + -o StrictHostKeyChecking=no + -o UserKnownHostsFile="${KNOWN_HOSTS_FILE}" + -o LogLevel=ERROR +) + +printf 'schema_version=windows99_vmx_source_backup_search_collector_v1\n' +printf 'target_host_alias=99\n' +printf 'target_vm_alias=111\n' +printf 'search_mode=filename_path_fingerprint_only\n' +printf 'search_hosts=%s\n' "$HOSTS" +printf 'search_roots_profile=linux_backup_roots\n' +printf 'max_depth=%s\n' "$MAX_DEPTH" +printf 'secret_value_read=false\n' +printf 'password_prompt_allowed=false\n' +printf 'file_content_read=false\n' +printf 'raw_path_output=false\n' +printf 'path_fingerprint_only=true\n' +printf 'remote_write_performed=false\n' +printf 'host_reboot_performed=false\n' +printf 'vm_power_change_performed=false\n' +printf 'windows_service_restart_performed=false\n' +printf 'windows_registry_apply_performed=false\n' +printf 'scheduled_task_write_performed=false\n' + +total_candidates=0 +searched_hosts=0 +blocked_hosts=0 + +IFS=',' read -r -a host_entries <<<"$HOSTS" +for entry in "${host_entries[@]}"; do + [[ -n "$entry" && "$entry" == *=* ]] || continue + alias_name="${entry%%=*}" + ssh_target="${entry#*=}" + remote_script=$(cat </dev/null || true; } \\ + | while IFS= read -r candidate_path; do + printf '%s' "\$candidate_path" | sha256sum | awk '{print \$1}' + done \\ + | sed -n '1,120p' +REMOTE +) + output="" + status="searched" + if output="$(timeout "${SSH_TIMEOUT}s" ssh "${SSH_OPTS[@]}" "$ssh_target" "bash -s" <<<"$remote_script" 2>/dev/null)"; then + searched_hosts=$((searched_hosts + 1)) + else + status="blocked_ssh_batchmode_unavailable" + blocked_hosts=$((blocked_hosts + 1)) + fi + candidate_count=0 + fingerprints="" + if [[ -n "$output" ]]; then + candidate_count="$(printf '%s\n' "$output" | sed '/^$/d' | wc -l | tr -d ' ')" + fingerprints="$(printf '%s\n' "$output" | sed '/^$/d' | paste -sd, -)" + fi + total_candidates=$((total_candidates + candidate_count)) + printf 'HOST_SEARCH alias=%s status=%s candidate_source_count=%s path_fingerprints=%s\n' \ + "$alias_name" "$status" "$candidate_count" "$fingerprints" +done + +printf 'searched_host_count=%s\n' "$searched_hosts" +printf 'blocked_host_count=%s\n' "$blocked_hosts" +printf 'candidate_source_count=%s\n' "$total_candidates" +if [[ "$total_candidates" -gt 0 ]]; then + printf 'search_status=candidate_fingerprint_found_requires_internal_path_resolution\n' + printf 'safe_next_step=resolve_candidate_fingerprint_to_authorized_internal_path_then_build_scoped_relink_dry_run_no_write\n' +else + printf 'search_status=blocked_no_verified_vmx_source_found\n' + printf 'safe_next_step=continue_verified_backup_inventory_search_or_authorized_console_source_path_then_scoped_dry_run_no_write\n' +fi diff --git a/scripts/reboot-recovery/tests/test_windows99_vmx_source_backup_search.py b/scripts/reboot-recovery/tests/test_windows99_vmx_source_backup_search.py new file mode 100644 index 000000000..0429638d6 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_windows99_vmx_source_backup_search.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import os +import subprocess +import textwrap +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ( + ROOT + / "scripts" + / "reboot-recovery" + / "collect-windows99-vmx-source-backup-search.sh" +) + + +def _write_executable(path: Path, text: str) -> None: + path.write_text(textwrap.dedent(text).lstrip()) + path.chmod(0o755) + + +def _key_values(stdout: str) -> dict[str, str]: + values: dict[str, str] = {} + for line in stdout.splitlines(): + if "=" not in line or line.startswith("HOST_SEARCH "): + continue + key, value = line.split("=", 1) + values[key] = value + return values + + +def test_backup_search_collector_is_no_secret_no_content_check_mode() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert "windows99_vmx_source_backup_search_collector_v1" in text + assert "BatchMode=yes" in text + assert "PreferredAuthentications=publickey" in text + assert "PasswordAuthentication=no" in text + assert "KbdInteractiveAuthentication=no" in text + assert "NumberOfPasswordPrompts=0" in text + assert "path_fingerprint_only=true" in text + assert "file_content_read=false" in text + assert "remote_write_performed=false" in text + for forbidden in [ + "sshpass", + "PasswordAuthentication=yes", + "KbdInteractiveAuthentication=yes", + "scp ", + "sudo ", + "cat \"$candidate_path\"", + "Get-Content", + "Start-VM", + "vmrun start", + "shutdown", + "Restart-Computer", + "Set-Content", + "Copy-Item", + "Move-Item", + "Remove-Item", + ]: + assert forbidden not in text + + +def test_backup_search_collector_emits_fingerprints_not_paths(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + _write_executable( + fake_bin / "ssh", + """ + #!/usr/bin/env bash + target="${@: -2:1}" + if [[ "$target" == *"blocked"* ]]; then + exit 255 + fi + printf '%s\n' 'fp-one' + printf '%s\n' 'fp-two' + """, + ) + + env = os.environ.copy() + env["PATH"] = f"{fake_bin}:{env['PATH']}" + env["WINDOWS99_VMX_BACKUP_SEARCH_KNOWN_HOSTS_FILE"] = str( + fake_bin / "known_hosts" + ) + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "--hosts", + "ok=wooo@ok,blocked=wooo@blocked", + "--timeout", + "3", + ], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0 + values = _key_values(result.stdout) + assert values["schema_version"] == "windows99_vmx_source_backup_search_collector_v1" + assert values["target_vm_alias"] == "111" + assert values["candidate_source_count"] == "2" + assert values["searched_host_count"] == "1" + assert values["blocked_host_count"] == "1" + assert "HOST_SEARCH alias=ok status=searched candidate_source_count=2" in result.stdout + assert "HOST_SEARCH alias=blocked status=blocked_ssh_batchmode_unavailable" in ( + result.stdout + ) + assert "path_fingerprints=fp-one,fp-two" in result.stdout + assert "192.168.0." not in result.stdout + assert "/home/wooo" not in result.stdout + assert "file_content_read=false" in result.stdout + assert "remote_write_performed=false" in result.stdout + assert "vm_power_change_performed=false" in result.stdout