From 813cab20145d8344ee930c83b87de993ac2099a4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 29 Jun 2026 19:57:23 +0800 Subject: [PATCH] feat(api): expose reboot recovery freshness readback --- .gitea/workflows/cd.yaml | 6 + apps/api/src/api/v1/agents.py | 41 +- .../services/delivery_closure_workbench.py | 161 ++++ .../reboot_auto_recovery_slo_scorecard.py | 178 +++++ .../test_delivery_closure_workbench_api.py | 85 +- ..._reboot_auto_recovery_slo_scorecard_api.py | 79 ++ docs/LOGBOOK.md | 26 + ...priority-work-order-readback.snapshot.json | 737 ++++++++++-------- ...-auto-recovery-slo-scorecard.snapshot.json | 148 +++- .../test_cd_controlled_runtime_profile.py | 4 + .../reboot-auto-recovery-slo-scorecard.py | 233 +++++- ...test_reboot_auto_recovery_slo_scorecard.py | 131 ++++ 12 files changed, 1497 insertions(+), 332 deletions(-) create mode 100644 apps/api/src/services/reboot_auto_recovery_slo_scorecard.py create mode 100644 apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index a05460a84..fce278e2e 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -262,6 +262,8 @@ jobs: ;; apps/api/src/services/gitea_private_inventory_p0_scorecard.py) ;; + apps/api/src/services/reboot_auto_recovery_slo_scorecard.py) + ;; apps/api/src/services/iwooos_security_operating_system.py) ;; apps/api/Dockerfile) @@ -328,6 +330,8 @@ jobs: ;; apps/api/tests/test_gitea_private_inventory_p0_scorecard_api.py) ;; + apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py) + ;; apps/api/tests/test_iwooos_security_operating_system.py) ;; apps/api/tests/e2e_network_test.py) @@ -492,6 +496,7 @@ jobs: src/services/credential_escrow_evidence_intake_readiness.py \ src/services/gitea_authenticated_inventory_payload_validation.py \ src/services/gitea_private_inventory_p0_scorecard.py \ + src/services/reboot_auto_recovery_slo_scorecard.py \ src/services/iwooos_security_operating_system.py \ src/services/awoooi_gitea_onboarding_warning_step_dashboard.py \ src/services/awoooi_gitea_onboarding_warning_step_owner_package.py \ @@ -530,6 +535,7 @@ jobs: tests/test_delivery_closure_workbench_api.py \ tests/test_credential_escrow_evidence_intake_readiness_api.py \ tests/test_gitea_private_inventory_p0_scorecard_api.py \ + tests/test_reboot_auto_recovery_slo_scorecard_api.py \ tests/test_iwooos_security_operating_system.py \ tests/e2e_network_test.py::TestHMACVerification::test_valid_hmac_signature \ tests/test_p0_cicd_baseline_source_readiness_api.py \ diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index d74669623..2376a819d 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -338,12 +338,12 @@ from src.services.dependency_upgrade_approval_package_template import ( from src.services.docker_build_surface_inventory import ( load_latest_docker_build_surface_inventory, ) -from src.services.gitea_private_inventory_p0_scorecard import ( - load_latest_gitea_private_inventory_p0_scorecard, -) from src.services.gitea_authenticated_inventory_payload_validation import ( validate_gitea_authenticated_inventory_payload, ) +from src.services.gitea_private_inventory_p0_scorecard import ( + load_latest_gitea_private_inventory_p0_scorecard, +) from src.services.gitea_workflow_runner_health import ( load_latest_gitea_workflow_runner_health, ) @@ -378,6 +378,9 @@ from src.services.product_code_review_gate import ( load_latest_product_code_review_gate, ) from src.services.public_redaction import redact_public_lan_topology +from src.services.reboot_auto_recovery_slo_scorecard import ( + load_latest_reboot_auto_recovery_slo_scorecard, +) from src.services.runtime_surface_inventory import ( load_latest_runtime_surface_inventory, ) @@ -1042,6 +1045,38 @@ async def get_p0_cicd_baseline_source_readiness() -> dict[str, Any]: ) from exc +@router.get( + "/reboot-auto-recovery-slo-scorecard", + response_model=dict[str, Any], + summary="取得 P0-006 reboot auto-recovery SLO scorecard", + description=( + "讀取已提交的 P0-006 reboot auto-recovery 10-minute SLO scorecard;" + "此端點只回傳 host boot probe、post-reboot readiness、StockPlatform freshness / " + "ingestion readback、retry window 與 controlled recovery gate 狀態。" + "它不重啟主機、不 restart service、不寫資料庫、不偽造 freshness marker、" + "不讀 secret、不觸發 workflow、不呼叫 GitHub。" + ), +) +async def get_reboot_auto_recovery_slo_scorecard() -> dict[str, Any]: + """回傳 P0-006 reboot auto-recovery SLO scorecard 只讀快照。""" + try: + payload = await asyncio.to_thread( + load_latest_reboot_auto_recovery_slo_scorecard + ) + return redact_public_lan_topology(payload) + except FileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except (json.JSONDecodeError, ValueError) as exc: + logger.error("reboot_auto_recovery_slo_scorecard_invalid", error=str(exc)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="P0-006 reboot auto-recovery SLO scorecard 快照無效", + ) from exc + + @router.get( "/gitea-private-inventory-p0-scorecard", response_model=dict[str, Any], diff --git a/apps/api/src/services/delivery_closure_workbench.py b/apps/api/src/services/delivery_closure_workbench.py index a07c490ec..970d91a2f 100644 --- a/apps/api/src/services/delivery_closure_workbench.py +++ b/apps/api/src/services/delivery_closure_workbench.py @@ -30,6 +30,9 @@ from src.services.gitea_workflow_runner_health import ( from src.services.p0_cicd_baseline_source_readiness import ( load_latest_p0_cicd_baseline_source_readiness, ) +from src.services.reboot_auto_recovery_slo_scorecard import ( + load_latest_reboot_auto_recovery_slo_scorecard, +) from src.services.runtime_surface_inventory import ( load_latest_runtime_surface_inventory, ) @@ -47,6 +50,7 @@ def load_delivery_closure_workbench() -> dict[str, Any]: runtime = load_latest_runtime_surface_inventory() backup = load_latest_backup_dr_readiness_matrix() credential_escrow_intake = load_latest_credential_escrow_evidence_intake_readiness() + reboot_slo = load_latest_reboot_auto_recovery_slo_scorecard() return build_delivery_closure_workbench( status_cleanup=status_cleanup, production_deploy=production_deploy, @@ -56,6 +60,7 @@ def load_delivery_closure_workbench() -> dict[str, Any]: runtime=runtime, backup=backup, credential_escrow_intake=credential_escrow_intake, + reboot_slo=reboot_slo, ) @@ -69,6 +74,7 @@ def build_delivery_closure_workbench( runtime: dict[str, Any], backup: dict[str, Any], credential_escrow_intake: dict[str, Any], + reboot_slo: dict[str, Any], ) -> dict[str, Any]: """Build the delivery workbench response from already validated snapshots.""" status_summary = _dict(status_cleanup.get("summary")) @@ -94,6 +100,13 @@ def build_delivery_closure_workbench( runtime_rollups = _dict(runtime.get("rollups")) backup_status = _dict(backup.get("program_status")) backup_rollups = _dict(backup.get("rollups")) + reboot_readback = _dict(reboot_slo.get("readback")) + reboot_rollups = _dict(reboot_slo.get("rollups")) + reboot_stockplatform = _dict(reboot_slo.get("stockplatform_data_freshness")) + reboot_stockplatform_eod = _dict(reboot_stockplatform.get("eod_window")) + reboot_stockplatform_recovery_gate = _dict( + reboot_stockplatform.get("controlled_recovery_gate") + ) credential_intake_rollups = _dict(credential_escrow_intake.get("rollups")) credential_intake_readback = _dict(credential_escrow_intake.get("readback")) single_preflight_intake = _dict( @@ -106,6 +119,7 @@ def build_delivery_closure_workbench( private_inventory_blockers = _int( private_inventory_rollups.get("active_blocker_count") ) + reboot_blockers = _int(reboot_rollups.get("active_blocker_count")) credential_escrow_required_items = _int( backup_rollups.get("credential_escrow_required_item_count") ) @@ -397,6 +411,96 @@ def build_delivery_closure_workbench( "href": "/deployments", "next_action": _first_string(production_deploy.get("next_actions")), }, + { + "id": "reboot_auto_recovery", + "source_id": "reboot_auto_recovery_slo_scorecard", + "completion_percent": _percent( + reboot_rollups.get("readiness_percent") + ), + "status": str(reboot_slo.get("status") or "unknown"), + "blocker_count": reboot_blockers, + "metric": { + "kind": "reboot_auto_recovery_slo", + "workplan_id": str(reboot_readback.get("workplan_id") or "P0-006"), + "target_minutes": _int(reboot_readback.get("target_minutes")), + "can_claim_all_services_recovered_within_target": reboot_rollups.get( + "can_claim_all_services_recovered_within_target" + ) + is True, + "active_blockers": _strings(reboot_slo.get("active_blockers")), + "post_start_blocked": _int( + reboot_rollups.get("post_start_blocked") + ), + "service_green": reboot_rollups.get("service_green") is True, + "product_data_green": reboot_rollups.get("product_data_green") + is True, + "backup_core_green": reboot_rollups.get("backup_core_green") is True, + "observed_host_count": _int( + reboot_rollups.get("observed_host_count") + ), + "missing_host_count": _int(reboot_rollups.get("missing_host_count")), + "unreachable_host_count": _int( + reboot_rollups.get("unreachable_host_count") + ), + "stale_host_count": _int(reboot_rollups.get("stale_host_count")), + "stockplatform_freshness_status": str( + reboot_rollups.get("stockplatform_freshness_status") or "" + ), + "stockplatform_ingestion_status": str( + reboot_rollups.get("stockplatform_ingestion_status") or "" + ), + "stockplatform_freshness_blocker_count": _int( + reboot_rollups.get("stockplatform_freshness_blocker_count") + ), + "stockplatform_ingestion_blocker_count": _int( + reboot_rollups.get("stockplatform_ingestion_blocker_count") + ), + "stockplatform_freshness_blockers": _strings( + reboot_stockplatform.get("freshness_blockers") + ), + "stockplatform_ingestion_blockers": _strings( + reboot_stockplatform.get("ingestion_blockers") + ), + "stockplatform_eod_classification": str( + reboot_stockplatform_eod.get("classification") or "" + ), + "stockplatform_eod_next_action": str( + reboot_stockplatform_eod.get("next_action") or "" + ), + "stockplatform_final_retry_window_end_local": str( + reboot_stockplatform_eod.get("final_retry_window_end_local") or "" + ), + "stockplatform_final_retry_window_passed": reboot_rollups.get( + "stockplatform_final_retry_window_passed" + ) + is True, + "stockplatform_controlled_recovery_gate_required": reboot_rollups.get( + "stockplatform_controlled_recovery_gate_required" + ) + is True, + "stockplatform_controlled_recovery_gate_status": str( + reboot_stockplatform_recovery_gate.get("status") or "" + ), + "host_reboot_performed": _dict( + reboot_slo.get("operation_boundaries") + ).get("host_reboot_performed") + is True, + "service_restart_performed": _dict( + reboot_slo.get("operation_boundaries") + ).get("service_restart_performed") + is True, + "database_write_or_restore_performed": _dict( + reboot_slo.get("operation_boundaries") + ).get("database_write_or_restore_performed") + is True, + "secret_value_collection_allowed": _dict( + reboot_slo.get("operation_boundaries") + ).get("secret_value_collection_allowed") + is True, + }, + "href": "/operations", + "next_action": str(reboot_readback.get("safe_next_step") or ""), + }, { "id": "credential_escrow", "source_id": "backup_dr_credential_escrow", @@ -773,6 +877,7 @@ def build_delivery_closure_workbench( source_statuses = [ _source_status("status_cleanup", status_cleanup), _source_status("production_deploy_readback", production_deploy), + _source_status("reboot_auto_recovery_slo_scorecard", reboot_slo), _source_status("gitea_private_inventory_p0_scorecard", private_inventory), _source_status("p0_cicd_baseline_source_readiness", cicd_baseline), _source_status("gitea_ci_cd", gitea), @@ -815,6 +920,46 @@ def build_delivery_closure_workbench( "visibility_change_authorized": False, "refs_sync_authorized": False, "workflow_trigger_authorized": False, + "reboot_auto_recovery_status": str(reboot_slo.get("status") or ""), + "reboot_auto_recovery_workplan_id": str( + reboot_readback.get("workplan_id") or "P0-006" + ), + "reboot_auto_recovery_readiness_percent": _int( + reboot_rollups.get("readiness_percent") + ), + "reboot_auto_recovery_active_blocker_count": reboot_blockers, + "reboot_auto_recovery_can_claim_slo": reboot_rollups.get( + "can_claim_all_services_recovered_within_target" + ) + is True, + "reboot_auto_recovery_service_green": reboot_rollups.get("service_green") + is True, + "reboot_auto_recovery_product_data_green": reboot_rollups.get( + "product_data_green" + ) + is True, + "reboot_auto_recovery_observed_host_count": _int( + reboot_rollups.get("observed_host_count") + ), + "reboot_auto_recovery_stale_host_count": _int( + reboot_rollups.get("stale_host_count") + ), + "reboot_auto_recovery_stockplatform_freshness_status": str( + reboot_rollups.get("stockplatform_freshness_status") or "" + ), + "reboot_auto_recovery_stockplatform_ingestion_status": str( + reboot_rollups.get("stockplatform_ingestion_status") or "" + ), + "reboot_auto_recovery_stockplatform_final_retry_window_passed": ( + reboot_rollups.get("stockplatform_final_retry_window_passed") is True + ), + "reboot_auto_recovery_stockplatform_controlled_recovery_gate_required": ( + reboot_rollups.get("stockplatform_controlled_recovery_gate_required") + is True + ), + "reboot_auto_recovery_safe_next_step": str( + reboot_readback.get("safe_next_step") or "" + ), "gitea_private_inventory_status": str(private_inventory.get("status") or ""), "gitea_private_inventory_workplan_id": str( private_inventory_readback.get("workplan_id") or "" @@ -1282,6 +1427,22 @@ def build_delivery_closure_workbench( "secret_value_collection_allowed": False, "backup_restore_execution_allowed": False, "active_scan_allowed": False, + "host_reboot_performed": _dict( + reboot_slo.get("operation_boundaries") + ).get("host_reboot_performed") + is True, + "service_restart_performed": _dict( + reboot_slo.get("operation_boundaries") + ).get("service_restart_performed") + is True, + "database_write_or_restore_performed": _dict( + reboot_slo.get("operation_boundaries") + ).get("database_write_or_restore_performed") + is True, + "stockplatform_manual_data_write_performed": _dict( + reboot_slo.get("operation_boundaries") + ).get("stockplatform_manual_data_write_performed") + is True, }, } diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py new file mode 100644 index 000000000..b359f0674 --- /dev/null +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -0,0 +1,178 @@ +"""P0-006 reboot auto-recovery SLO scorecard readback. + +This loader promotes the committed reboot recovery scorecard into a stable API +contract. It does not reboot hosts, restart services, query runtime secrets, or +write StockPlatform data; it only reads the committed JSON scorecard. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from src.services.snapshot_paths import default_operations_dir + +_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__)) +_SCORECARD_FILE = "awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json" +_SOURCE_SCHEMA_VERSION = "awoooi_reboot_auto_recovery_slo_scorecard_v1" +_API_SCHEMA_VERSION = "reboot_auto_recovery_slo_scorecard_readback_v1" + + +def load_latest_reboot_auto_recovery_slo_scorecard( + operations_dir: Path | None = None, +) -> dict[str, Any]: + """Load and validate the committed P0-006 reboot recovery scorecard.""" + directory = operations_dir or _DEFAULT_OPERATIONS_DIR + path = directory / _SCORECARD_FILE + with path.open(encoding="utf-8") as handle: + scorecard = json.load(handle) + + if not isinstance(scorecard, dict): + raise ValueError(f"{path}: expected JSON object") + if scorecard.get("schema_version") != _SOURCE_SCHEMA_VERSION: + raise ValueError(f"{path}: unsupported schema_version={scorecard.get('schema_version')!r}") + + payload = _build_payload(scorecard, path) + _require_operation_boundaries(payload, str(path)) + return payload + + +def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: + host_boot_detection = _dict(scorecard.get("host_boot_detection")) + post_reboot_readiness = _dict(scorecard.get("post_reboot_readiness")) + stockplatform = _dict(scorecard.get("stockplatform_data_freshness")) + active_blockers = _strings(scorecard.get("active_blockers")) + required_checks = { + "source_controls_present": all(_dict(scorecard.get("source_controls")).values()), + "required_hosts_observed": not _strings(host_boot_detection.get("missing_hosts")), + "required_hosts_reachable": not _strings(host_boot_detection.get("unreachable_hosts")), + "service_green": post_reboot_readiness.get("service_green") is True, + "product_data_green": post_reboot_readiness.get("product_data_green") is True, + "backup_core_green": post_reboot_readiness.get("backup_core_green") is True, + "host_188_service_green": post_reboot_readiness.get("host_188_service_green") is True, + "stockplatform_freshness_ok": stockplatform.get("freshness_status") == "ok", + "stockplatform_ingestion_ok": stockplatform.get("ingestion_status") in {"ok", "unknown"}, + "fresh_reboot_window_observed": not _strings(host_boot_detection.get("stale_hosts")), + "can_claim_slo": scorecard.get("can_claim_all_services_recovered_within_target") + is True, + } + completed_check_count = sum(1 for value in required_checks.values() if value) + readiness_percent = _percent( + completed_check_count / max(len(required_checks), 1) * 100 + ) + recovery_gate = _dict(stockplatform.get("controlled_recovery_gate")) + return { + "schema_version": _API_SCHEMA_VERSION, + "generated_at": str(scorecard.get("generated_at") or ""), + "priority": "P0-006", + "scope": "reboot_auto_recovery_slo_scorecard", + "status": str(scorecard.get("status") or "unknown"), + "readback": { + "workplan_id": "P0-006", + "workplan_title": "主機重啟自動偵測、自動觸發與 10 分鐘恢復 SLO", + "source_scorecard_ref": f"docs/operations/{path.name}", + "target_minutes": _int(scorecard.get("target_minutes")), + "safe_next_step": str(scorecard.get("safe_next_step") or ""), + }, + "host_boot_detection": host_boot_detection, + "post_reboot_readiness": post_reboot_readiness, + "stockplatform_data_freshness": stockplatform, + "active_blockers": active_blockers, + "required_checks": required_checks, + "rollups": { + "active_blocker_count": len(active_blockers), + "readiness_percent": readiness_percent, + "completed_check_count": completed_check_count, + "required_check_count": len(required_checks), + "can_claim_all_services_recovered_within_target": scorecard.get( + "can_claim_all_services_recovered_within_target" + ) + is True, + "observed_host_count": len(_strings(host_boot_detection.get("observed_hosts"))), + "missing_host_count": len(_strings(host_boot_detection.get("missing_hosts"))), + "unreachable_host_count": len( + _strings(host_boot_detection.get("unreachable_hosts")) + ), + "stale_host_count": len(_strings(host_boot_detection.get("stale_hosts"))), + "post_start_blocked": _int(post_reboot_readiness.get("post_start_blocked")), + "service_green": post_reboot_readiness.get("service_green") is True, + "product_data_green": post_reboot_readiness.get("product_data_green") is True, + "backup_core_green": post_reboot_readiness.get("backup_core_green") is True, + "stockplatform_freshness_status": str( + stockplatform.get("freshness_status") or "unknown" + ), + "stockplatform_ingestion_status": str( + stockplatform.get("ingestion_status") or "unknown" + ), + "stockplatform_freshness_blocker_count": len( + _strings(stockplatform.get("freshness_blockers")) + ), + "stockplatform_ingestion_blocker_count": len( + _strings(stockplatform.get("ingestion_blockers")) + ), + "stockplatform_final_retry_window_passed": _dict( + stockplatform.get("eod_window") + ).get("final_retry_window_passed") + is True, + "stockplatform_controlled_recovery_gate_required": recovery_gate.get( + "required" + ) + is True, + }, + "operation_boundaries": { + "read_only_api_allowed": True, + "host_reboot_performed": False, + "host_restart_performed": False, + "service_restart_performed": False, + "database_write_or_restore_performed": False, + "stockplatform_manual_data_write_performed": False, + "secret_value_collection_allowed": False, + "github_api_used": False, + "workflow_trigger_performed": False, + "runtime_write_allowed": False, + }, + } + + +def _require_operation_boundaries(payload: dict[str, Any], label: str) -> None: + boundaries = _dict(payload.get("operation_boundaries")) + forbidden_true = [ + key + for key in ( + "host_reboot_performed", + "host_restart_performed", + "service_restart_performed", + "database_write_or_restore_performed", + "stockplatform_manual_data_write_performed", + "secret_value_collection_allowed", + "github_api_used", + "workflow_trigger_performed", + "runtime_write_allowed", + ) + if boundaries.get(key) is True + ] + if forbidden_true: + raise ValueError(f"{label}: forbidden operation boundary true: {forbidden_true}") + + +def _dict(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _int(value: Any) -> int: + if isinstance(value, bool): + return int(value) + if isinstance(value, int | float): + return int(value) + return 0 + + +def _percent(value: Any) -> int: + return max(0, min(100, round(float(value or 0)))) + + +def _strings(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if item is not None] diff --git a/apps/api/tests/test_delivery_closure_workbench_api.py b/apps/api/tests/test_delivery_closure_workbench_api.py index f4a9cbdff..81341fcab 100644 --- a/apps/api/tests/test_delivery_closure_workbench_api.py +++ b/apps/api/tests/test_delivery_closure_workbench_api.py @@ -159,10 +159,48 @@ def test_delivery_closure_workbench_exposes_p0_005_credential_escrow_lane(): ) +def test_delivery_closure_workbench_exposes_p0_006_reboot_slo_lane(): + payload = load_delivery_closure_workbench() + + _assert_delivery_workbench_shape(payload) + lane = {lane["id"]: lane for lane in payload["lanes"]}["reboot_auto_recovery"] + + assert lane["source_id"] == "reboot_auto_recovery_slo_scorecard" + assert lane["status"] == "blocked_reboot_auto_recovery_slo_not_ready" + assert lane["blocker_count"] == 6 + assert lane["completion_percent"] == 45 + assert lane["metric"]["kind"] == "reboot_auto_recovery_slo" + assert lane["metric"]["workplan_id"] == "P0-006" + assert lane["metric"]["target_minutes"] == 10 + assert lane["metric"]["can_claim_all_services_recovered_within_target"] is False + assert lane["metric"]["observed_host_count"] == 4 + assert lane["metric"]["missing_host_count"] == 0 + assert lane["metric"]["unreachable_host_count"] == 0 + assert lane["metric"]["stale_host_count"] == 4 + assert lane["metric"]["service_green"] is False + assert lane["metric"]["product_data_green"] is False + assert lane["metric"]["backup_core_green"] is True + assert lane["metric"]["stockplatform_freshness_status"] == "blocked" + assert lane["metric"]["stockplatform_ingestion_status"] == "blocked" + assert lane["metric"]["stockplatform_freshness_blocker_count"] == 2 + assert lane["metric"]["stockplatform_ingestion_blocker_count"] == 1 + assert lane["metric"]["stockplatform_final_retry_window_passed"] is False + assert lane["metric"]["stockplatform_controlled_recovery_gate_required"] is False + assert lane["metric"]["host_reboot_performed"] is False + assert lane["metric"]["service_restart_performed"] is False + assert lane["metric"]["database_write_or_restore_performed"] is False + assert lane["metric"]["secret_value_collection_allowed"] is False + assert "stockplatform_freshness_blocked" in lane["metric"]["active_blockers"] + assert lane["next_action"] == ( + "inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_" + "rerun_slo_verify_only_no_reboot" + ) + + def _assert_delivery_workbench_shape(data: dict): assert data["schema_version"] == "delivery_closure_workbench_v1" - assert data["summary"]["source_count"] == 7 - assert data["summary"]["loaded_source_count"] == 7 + assert data["summary"]["source_count"] == 8 + assert data["summary"]["loaded_source_count"] == 8 assert data["summary"]["runtime_execution_authorized"] is False assert data["summary"]["remote_write_authorized"] is False assert data["summary"]["repo_creation_authorized"] is False @@ -174,6 +212,41 @@ def _assert_delivery_workbench_shape(data: dict): assert data["summary"]["github_lane_excluded_from_p0_blocker_count"] is True assert data["summary"]["github_blocked_preflight_target_count"] == 0 assert data["summary"]["github_operator_unblock_required"] is False + assert data["summary"]["reboot_auto_recovery_status"] == ( + "blocked_reboot_auto_recovery_slo_not_ready" + ) + assert data["summary"]["reboot_auto_recovery_workplan_id"] == "P0-006" + assert data["summary"]["reboot_auto_recovery_readiness_percent"] == 45 + assert data["summary"]["reboot_auto_recovery_active_blocker_count"] == 6 + assert data["summary"]["reboot_auto_recovery_can_claim_slo"] is False + assert data["summary"]["reboot_auto_recovery_service_green"] is False + assert data["summary"]["reboot_auto_recovery_product_data_green"] is False + assert data["summary"]["reboot_auto_recovery_observed_host_count"] == 4 + assert data["summary"]["reboot_auto_recovery_stale_host_count"] == 4 + assert ( + data["summary"]["reboot_auto_recovery_stockplatform_freshness_status"] + == "blocked" + ) + assert ( + data["summary"]["reboot_auto_recovery_stockplatform_ingestion_status"] + == "blocked" + ) + assert ( + data["summary"][ + "reboot_auto_recovery_stockplatform_final_retry_window_passed" + ] + is False + ) + assert ( + data["summary"][ + "reboot_auto_recovery_stockplatform_controlled_recovery_gate_required" + ] + is False + ) + assert data["summary"]["reboot_auto_recovery_safe_next_step"] == ( + "inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_" + "rerun_slo_verify_only_no_reboot" + ) assert data["summary"]["gitea_private_inventory_status"] == ( "blocked_waiting_gitea_authenticated_or_owner_export_inventory" ) @@ -317,6 +390,7 @@ def _assert_delivery_workbench_shape(data: dict): assert lane_ids == { "release", "production_deploy", + "reboot_auto_recovery", "credential_escrow", "gitea_private_inventory", "cicd_baseline", @@ -341,3 +415,10 @@ def _assert_delivery_workbench_shape(data: dict): assert data["operation_boundaries"]["github_write_channel_ready"] is False assert data["operation_boundaries"]["github_controlled_apply_allowed"] is False assert data["operation_boundaries"]["secret_value_collection_allowed"] is False + assert data["operation_boundaries"]["host_reboot_performed"] is False + assert data["operation_boundaries"]["service_restart_performed"] is False + assert data["operation_boundaries"]["database_write_or_restore_performed"] is False + assert ( + data["operation_boundaries"]["stockplatform_manual_data_write_performed"] + is False + ) diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py new file mode 100644 index 000000000..bad059ec3 --- /dev/null +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.api.v1.agents import router +from src.services.reboot_auto_recovery_slo_scorecard import ( + load_latest_reboot_auto_recovery_slo_scorecard, +) + + +def test_reboot_auto_recovery_slo_scorecard_loader_exposes_stockplatform_gate(): + payload = load_latest_reboot_auto_recovery_slo_scorecard() + + _assert_reboot_slo_payload(payload) + + +def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(): + app = FastAPI() + app.include_router(router, prefix="/api/v1") + client = TestClient(app) + + response = client.get("/api/v1/agents/reboot-auto-recovery-slo-scorecard") + + assert response.status_code == 200 + _assert_reboot_slo_payload(response.json()) + + +def _assert_reboot_slo_payload(payload: dict): + assert payload["schema_version"] == "reboot_auto_recovery_slo_scorecard_readback_v1" + assert payload["priority"] == "P0-006" + assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready" + assert payload["readback"]["workplan_id"] == "P0-006" + assert payload["readback"]["target_minutes"] == 10 + assert payload["readback"]["safe_next_step"] == ( + "inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_" + "rerun_slo_verify_only_no_reboot" + ) + assert payload["rollups"]["active_blocker_count"] == 6 + assert payload["rollups"]["readiness_percent"] == 45 + assert payload["rollups"]["observed_host_count"] == 4 + assert payload["rollups"]["missing_host_count"] == 0 + assert payload["rollups"]["unreachable_host_count"] == 0 + assert payload["rollups"]["stale_host_count"] == 4 + assert payload["rollups"]["service_green"] is False + assert payload["rollups"]["product_data_green"] is False + assert payload["rollups"]["backup_core_green"] is True + assert payload["rollups"]["stockplatform_freshness_status"] == "blocked" + assert payload["rollups"]["stockplatform_ingestion_status"] == "blocked" + assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 2 + assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 1 + assert payload["rollups"]["stockplatform_final_retry_window_passed"] is False + assert ( + payload["rollups"]["stockplatform_controlled_recovery_gate_required"] + is False + ) + assert "stockplatform_freshness_blocked" in payload["active_blockers"] + stockplatform = payload["stockplatform_data_freshness"] + assert stockplatform["freshness_endpoint_readback_present"] is True + assert stockplatform["ingestion_endpoint_readback_present"] is True + assert stockplatform["freshness_blockers"] == [ + "core_margin_short_daily_missing", + "ai_recommendations_stale", + ] + assert stockplatform["ingestion_blockers"] == [ + "core.margin_short_daily_incomplete" + ] + assert stockplatform["eod_window"]["final_retry_window_passed"] is False + assert stockplatform["controlled_recovery_gate"]["required"] is False + assert "manual_db_update" in stockplatform["controlled_recovery_gate"][ + "forbidden_actions" + ] + boundaries = payload["operation_boundaries"] + assert boundaries["host_reboot_performed"] is False + assert boundaries["service_restart_performed"] is False + assert boundaries["database_write_or_restore_performed"] is False + assert boundaries["stockplatform_manual_data_write_performed"] is False + assert boundaries["secret_value_collection_allowed"] is False + assert boundaries["github_api_used"] is False diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index a2b214a3c..54aa9aea7 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -49584,6 +49584,32 @@ production browser smoke: - 沒有重啟主機,沒有 restart Docker / Nginx / K3s / DB / firewall,沒有修改 runner/host 權限。 - 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +## 2026-06-29 — 19:55 P0-006 StockPlatform freshness / ingestion readback 接入 SLO scorecard + +**完成內容**: +- 將 P0-006 `reboot-auto-recovery-slo-scorecard.py` 擴充為可讀 StockPlatform `/api/v1/system/freshness` 與 `/api/v1/system/ingestion` JSON readback;scorecard 現在直接輸出 freshness blockers、ingestion blockers、blocked sources、latest source runs、EOD retry window 與 controlled recovery gate。 +- 新增 `GET /api/v1/agents/reboot-auto-recovery-slo-scorecard` 只讀 API;此端點只讀 committed scorecard,不重啟主機、不 restart service、不寫 DB、不讀 secret、不觸發 workflow、不呼叫 GitHub。 +- Delivery Workbench 新增 `reboot_auto_recovery` lane,將 P0-006 放回第一級交付主線;summary 同步顯示 `active_blocker_count=6`、`readiness_percent=45`、`observed_host_count=4`、StockPlatform freshness / ingestion blocked 與 final retry window 尚未通過。 +- 更新 P0 ordered priority snapshot:目前 P0-006 的唯一下一步是等 23:35 final retry window 後 rerun verify-only;若仍 blocked,才開 controlled data recovery gate,且必須具備 source diff、check-mode / dry-run、rollback 與 post-freshness verifier。 +- 補 `.gitea/workflows/cd.yaml` controlled-runtime 白名單與 `ops/runner/test_cd_controlled_runtime_profile.py`,避免新增 P0-006 API service / test 後誤掉進 full / B5 profile。 + +**live readback**: +- StockPlatform healthz / API healthz:HTTP 200。 +- Freshness:`status=blocked`、`latest_trading_date=2026-06-29`、blockers=`core_margin_short_daily_missing, ai_recommendations_stale`。 +- Ingestion:`status=blocked`、blocker=`core.margin_short_daily_incomplete`。 +- 目前時間仍在 final retry window 前,`stockplatform_controlled_recovery_gate_required=false`;safe next step 為 `inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_rerun_slo_verify_only_no_reboot`。 + +**本地驗證結果**: +- `python3.11 -m py_compile`:P0-006 scorecard、API service、Delivery Workbench、agents router 與相關 tests 通過。 +- Focused pytest:P0-006 scorecard / API / Delivery Workbench / CD profile `26 passed`。 +- `guard-gitea-runner-pressure.py`:通過,`auto_branch_events_on_110=0`、`generic_runner_labels=0`。 +- `check-gitea-step-env-secrets.js`:通過。 +- `git diff --check`:通過。 + +**仍維持**: +- 沒有重啟任何主機;沒有 restart Docker / Nginx / K3s / DB / firewall;沒有手動寫 StockPlatform DB;沒有 fake freshness marker。 +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth;沒有使用 GitHub / gh / GitHub API / GitHub Actions。 + ## 2026-06-29 — 15:34 P0-005 credential escrow readiness safe next step source fix **照優先順序完成的讀回**: diff --git a/docs/operations/awoooi-priority-work-order-readback.snapshot.json b/docs/operations/awoooi-priority-work-order-readback.snapshot.json index 02b0f01c3..d29b05949 100644 --- a/docs/operations/awoooi-priority-work-order-readback.snapshot.json +++ b/docs/operations/awoooi-priority-work-order-readback.snapshot.json @@ -1,204 +1,283 @@ { - "schema_version": "awoooi_priority_work_order_readback_v1", - "generated_at": "2026-06-29T19:21:27+08:00", - "status": "p0_ordered_readback_p0_006_stock_freshness_blocked_p0_005_refs_waiting_p0_003_payload_validator_ready", - "source_refs": { - "global_scorecard": "~/.codex/product-runtime-governance-completion-scorecard.snapshot.json", - "workstation_dashboard": "~/.codex/codex-workstation-sync-dashboard.snapshot.json", - "post_reboot_summary": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260629-191950/summary.txt", - "full_stack_cold_start_check": "scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color", - "delivery_closure_workbench": "https://awoooi.wooo.work/api/v1/agents/delivery-closure-workbench", - "public_gitea_queue_readback": "ops/runner/read-public-gitea-actions-queue.py --json", - "credential_escrow_scorecard": "/tmp/awoooi-credential-escrow-intake-scorecard-20260629-1200-priority.json", - "dr_escrow_evidence_checklist_generator": "scripts/reboot-recovery/dr-escrow-evidence-checklist.py", - "gitea_private_inventory_p0_scorecard": "docs/operations/awoooi-gitea-private-inventory-p0-scorecard.snapshot.json", - "reboot_auto_recovery_slo_scorecard": "docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json", - "reboot_auto_recovery_slo_metric": "/home/wooo/node_exporter_textfiles/reboot_auto_recovery_slo.prom", - "credential_escrow_live_status": "/tmp/awoooi-p0-005-escrow-status-live.txt", - "dr_escrow_evidence_checklist_latest": "/tmp/awoooi-dr-escrow-evidence-checklist-current.json", - "gitea_repo_inventory_snapshot": "docs/security/gitea-repo-inventory.snapshot.json", - "credential_escrow_intake_readiness_api": "/api/v1/agents/credential-escrow-evidence-intake-readiness", - "credential_escrow_intake_readiness_service": "apps/api/src/services/credential_escrow_evidence_intake_readiness.py", - "credential_escrow_intake_readiness_tests": "apps/api/tests/test_credential_escrow_evidence_intake_readiness_api.py", - "gitea_authenticated_inventory_payload_validator": "scripts/security/gitea-authenticated-inventory-payload-validator.py", - "gitea_authenticated_inventory_payload_validation_snapshot": "docs/operations/awoooi-gitea-authenticated-inventory-payload-validation.snapshot.json", - "stockplatform_freshness_readback": "https://stock.wooo.work/api/v1/system/freshness", - "stockplatform_ingestion_readback": "https://stock.wooo.work/api/v1/system/ingestion", - "owner_requested_openclaw_live_ops_space": "2026-06-29 owner request: Gather-like OpenClaw continuous animated live ops workspace" - }, - "current_head": { - "gitea_main_sha": "85ac9ec5896fc28d8b9a4ab915db578dcd861645", - "latest_successful_deploy_marker": "95bbc6bea chore(cd): deploy 5981586 [skip ci]", - "latest_successful_deployed_source_sha": "5981586d0f2aa3c1baa00630df8121fab2126008", - "latest_source_readiness_commit_sha": "85ac9ec5896fc28d8b9a4ab915db578dcd861645", - "latest_source_readiness_cd_run_id": null, - "latest_source_readiness_cd_run_status": "not_visible_in_public_queue_at_2026-06-29T19:20:13+08:00", - "source_readiness_ci_fix_required": false, - "no_matching_runner_visible": false, - "latest_verified_worktree_base_sha": "85ac9ec58", - "latest_fetched_gitea_main_subject": "fix(cd): keep gitea inventory validation on controlled profile" - }, "completed_in_priority_order": [ { - "workplan_id": "P0-001", - "title": "建立主機 runtime inventory 權威資料", - "status": "core_green_with_warnings", "evidence": { + "full_stack_blocked": 0, + "full_stack_pass": 91, + "full_stack_warn": 2, + "host_188_hygiene_blocked": false, "hosts_checked": [ "192.168.0.110", "192.168.0.120", "192.168.0.121", "192.168.0.188" ], - "full_stack_pass": 91, - "full_stack_warn": 2, - "full_stack_blocked": 0, + "post_start_blocked": 0, "post_start_pass": 43, "post_start_warn": 5, - "post_start_blocked": 0, + "runtime_action_authorized": false, "service_green": true, - "host_188_hygiene_blocked": false, - "wazuh_manager_registry_accepted": 6, - "runtime_action_authorized": false + "wazuh_manager_registry_accepted": 6 }, + "status": "core_green_with_warnings", + "title": "建立主機 runtime inventory 權威資料", "warnings_kept_visible": [ "110 systemd failed units remain", "188 momo daily sales stale but source preflight has no hard blocker" - ] + ], + "workplan_id": "P0-001" }, { - "workplan_id": "P0-004", - "title": "補 dev / prod CI/CD baseline", - "status": "production_deploy_closure_verified", "evidence": { - "non110_runner_ready": true, "latest_cd_run_id": "3868", "latest_cd_run_status": "Success", + "non110_runner_ready": true, "production_deploy_status": "closure_verified", - "production_image_tag_matches_main": true, "production_governance_fields_present": true, + "production_image_tag_matches_main": true, "runner_pressure_guard_ok": true - } - }, - { - "workplan_id": "P0-004-source-readiness", - "title": "P0-004 CI/CD baseline source readiness", - "status": "ready_for_template_copy_apply_gate", - "evidence": { - "required_source_count": 11, - "present_required_source_count": 11, - "missing_required_source_count": 0, - "source_readiness_percent": 100, - "blocked_source_ids": [] }, - "safe_next_step": "open_template_copy_apply_gate_after_source_readiness_green" + "status": "production_deploy_closure_verified", + "title": "補 dev / prod CI/CD baseline", + "workplan_id": "P0-004" }, { - "workplan_id": "P0-002", - "title": "建立 product.awoooi.yaml 產品 manifest 標準", - "status": "ready_for_product_manifest_adoption", "evidence": { + "blocked_source_ids": [], + "missing_required_source_count": 0, + "present_required_source_count": 11, + "required_source_count": 11, + "source_readiness_percent": 100 + }, + "safe_next_step": "open_template_copy_apply_gate_after_source_readiness_green", + "status": "ready_for_template_copy_apply_gate", + "title": "P0-004 CI/CD baseline source readiness", + "workplan_id": "P0-004-source-readiness" + }, + { + "evidence": { + "github_status": "stopped_retired_do_not_use", "manifest_present": true, "manifest_schema_present": true, "manifest_schema_version": "product_awoooi_manifest_v1", - "product_id": "awoooi", - "source_control_authority": "gitea", - "github_status": "stopped_retired_do_not_use", - "required_section_count": 5, - "present_required_section_count": 5, - "missing_required_section_count": 0, - "required_contract_ref_count": 3, - "present_contract_ref_count": 3, "missing_contract_ref_count": 0, + "missing_required_section_count": 0, + "present_contract_ref_count": 3, + "present_required_section_count": 5, + "product_id": "awoooi", + "required_contract_ref_count": 3, + "required_section_count": 5, + "source_control_authority": "gitea", "source_readiness_percent": 100 }, - "safe_next_step": "adopt_product_manifest_schema_for_remaining_products_without_repo_creation_or_secret_read" + "safe_next_step": "adopt_product_manifest_schema_for_remaining_products_without_repo_creation_or_secret_read", + "status": "ready_for_product_manifest_adoption", + "title": "建立 product.awoooi.yaml 產品 manifest 標準", + "workplan_id": "P0-002" } ], + "current_head": { + "gitea_main_sha": "b9293b76b56cd327f34b4f2fb723674665f69a1c", + "latest_fetched_gitea_main_subject": "feat(api): validate gitea owner attestation intake", + "latest_source_readiness_cd_run_id": null, + "latest_source_readiness_cd_run_status": "not_visible_in_public_queue_at_2026-06-29T19:20:13+08:00", + "latest_source_readiness_commit_sha": "85ac9ec5896fc28d8b9a4ab915db578dcd861645", + "latest_successful_deploy_marker": "95bbc6bea chore(cd): deploy 5981586 [skip ci]", + "latest_successful_deployed_source_sha": "5981586d0f2aa3c1baa00630df8121fab2126008", + "latest_verified_worktree_base_sha": "b9293b76", + "no_matching_runner_visible": false, + "source_readiness_ci_fix_required": false + }, + "generated_at": "2026-06-29T19:53:33+08:00", "in_progress_or_blocked_in_priority_order": [ { - "workplan_id": "P0-006", - "title": "主機重啟自動偵測、自動觸發與 10 分鐘恢復 SLO", - "status": "blocked_stockplatform_product_data_freshness_after_first_eod_window", - "reason": "Verify-only readback confirms the 110 reboot SLO timer remains enabled/active and all required hosts are observed. The current P0 blocker is no longer only an old boot window: after the 19:15 first EOD window, StockPlatform product data freshness is still blocked by core_margin_short_daily_missing and ai_recommendations_stale, so SERVICE_GREEN=0 and the 10-minute recovery claim is false until retry windows clear or a controlled data recovery gate is opened.", "evidence": { - "target_minutes": 10, - "can_claim_all_services_recovered_within_target": false, - "source_controls_added": true, - "host_boot_probe_source_present": true, - "slo_systemd_timer_source_present": true, - "slo_exporter_source_present": true, - "post_start_blocked": 1, - "service_green": false, - "product_data_green": false, - "backup_core_green": true, - "wazuh_dashboard_degraded": false, + "active_blockers": [ + "host_boot_observation_older_than_target_window", + "post_start_blocked_not_zero", + "product_data_green_not_1", + "service_green_not_1", + "stockplatform_freshness_blocked", + "stockplatform_ingestion_blocked" + ], "all_host_reboot_detection_missing": false, + "backup_core_green": true, + "can_claim_all_services_recovered_within_target": false, + "host_188_service_green": true, "host_boot_probe_missing_hosts": false, - "local_disk_free_gib_after_cleanup": 145.514, - "slo_installer_source_present": true, - "live_slo_timer_enabled": true, - "live_slo_timer_active": true, - "live_slo_service_last_result": "success", + "host_boot_probe_source_present": true, + "latest_live_slo_artifact": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260629-191950", "live_slo_metric_present": true, + "live_slo_service_last_result": "success", + "live_slo_timer_active": true, + "live_slo_timer_enabled": true, + "local_disk_free_gib_after_cleanup": 145.514, + "max_observed_uptime_seconds": 534664, + "missing_hosts": [], "observed_hosts": [ "110", "120", "121", "188" ], - "missing_hosts": [], - "unreachable_hosts": [], + "post_start_blocked": 1, + "product_data_green": false, + "safe_next_step_from_scorecard": "inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_rerun_slo_verify_only_no_reboot", + "service_green": false, + "slo_exporter_source_present": true, + "slo_installer_source_present": true, + "slo_systemd_timer_source_present": true, + "source_controls_added": true, "stale_hosts": [ "110", "120", "121", "188" ], - "max_observed_uptime_seconds": 534664, - "active_blockers": [ - "host_boot_observation_older_than_target_window", - "post_start_blocked_not_zero", - "product_data_green_not_1", - "service_green_not_1" + "stock_blocked_sources": [ + { + "latest_date": "2026-06-26", + "notes": "官方來源狀態:上櫃官方尚未發布,回傳日期 2026-06-29,來源列數 0;上市官方尚未發布,回傳日期 尚無,來源列數 0", + "row_count": 0, + "source": "core.margin_short_daily", + "status": "missing" + }, + { + "latest_date": "2026-06-26", + "notes": "最新日期 2026-06-26 早於最新交易日 2026-06-29", + "row_count": 2988, + "source": "ai.recommendations", + "status": "stale" + } ], - "host_188_service_green": true, - "safe_next_step_from_scorecard": "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_rerun_scorecard_until_status_slo_ready", - "latest_live_slo_artifact": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260629-191950", - "stockplatform_healthz_green": true, - "stockplatform_api_healthz_green": true, - "stockplatform_transient_502_before_success_seen": true, - "stock_freshness_status": "blocked", - "stock_latest_trading_date": "2026-06-29", "stock_blockers": [ "core_margin_short_daily_missing", "ai_recommendations_stale" ], - "stock_eod_window_pending": false, "stock_eod_classification": "after_first_eod_window_blocked", - "stock_eod_next_action": "inspect_ingestion_logs_and_wait_retry_windows", - "stock_eod_first_full_window_end_local": "19:15", "stock_eod_final_retry_window_end_local": "23:35", - "stock_official_margin_short_source_runs": [ + "stock_eod_final_retry_window_passed": false, + "stock_eod_first_full_window_end_local": "19:15", + "stock_eod_next_action": "inspect_ingestion_logs_and_wait_retry_windows", + "stock_eod_window_pending": false, + "stock_freshness_status": "blocked", + "stock_ingestion_blockers": [ + "core.margin_short_daily_incomplete" + ], + "stock_latest_source_runs": [ { - "source_run_id": 3384, + "finished_at": "2026-06-29T11:05:17.201062Z", "source_name": "official_margin_short_daily", - "target_date": "2026-06-29", + "source_run_id": 3384, + "started_at": "2026-06-29T11:05:17.201062Z", "status": "official_pending", - "started_at": "2026-06-29T11:05:17.201062Z" + "target_date": "2026-06-29" }, { - "source_run_id": 3383, + "finished_at": "2026-06-29T11:05:17.201062Z", "source_name": "official_margin_short_daily", - "target_date": "2026-06-29", + "source_run_id": 3383, + "started_at": "2026-06-29T11:05:17.201062Z", "status": "official_pending", - "started_at": "2026-06-29T11:05:17.201062Z" + "target_date": "2026-06-29" + }, + { + "finished_at": "2026-06-29T11:00:11.518268Z", + "source_name": "intelligence_security_linker", + "source_run_id": 3382, + "started_at": "2026-06-29T11:00:11.518268Z", + "status": "succeeded", + "target_date": null + }, + { + "finished_at": "2026-06-29T11:00:10.289509Z", + "source_name": "intelligence_reports_import", + "source_run_id": 3381, + "started_at": "2026-06-29T11:00:10.289509Z", + "status": "succeeded", + "target_date": null + }, + { + "finished_at": "2026-06-29T10:22:15.262488Z", + "source_name": "year_to_date_stock_backfill", + "source_run_id": 3380, + "started_at": "2026-06-29T10:22:15.262488Z", + "status": "planned", + "target_date": null + }, + { + "finished_at": "2026-06-29T10:22:13.464934Z", + "source_name": "official_margin_short_daily", + "source_run_id": 3378, + "started_at": "2026-06-29T10:22:13.464934Z", + "status": "official_pending", + "target_date": "2026-06-29" + }, + { + "finished_at": "2026-06-29T10:22:13.464934Z", + "source_name": "official_margin_short_daily", + "source_run_id": 3379, + "started_at": "2026-06-29T10:22:13.464934Z", + "status": "official_pending", + "target_date": "2026-06-29" + }, + { + "finished_at": "2026-06-29T10:21:56.651793Z", + "source_name": "community_etfinfo_holdings_daily", + "source_run_id": 3345, + "started_at": "2026-06-29T10:21:56.651793Z", + "status": "succeeded", + "target_date": "2026-06-26" + }, + { + "finished_at": "2026-06-29T10:21:56.651793Z", + "source_name": "community_etfinfo_active_changes_daily", + "source_run_id": 3346, + "started_at": "2026-06-29T10:21:56.651793Z", + "status": "succeeded", + "target_date": "2026-06-26" + }, + { + "finished_at": "2026-06-29T10:21:56.651793Z", + "source_name": "community_etfinfo_holdings_daily", + "source_run_id": 3343, + "started_at": "2026-06-29T10:21:56.651793Z", + "status": "succeeded", + "target_date": "2026-06-26" } - ] + ], + "stock_latest_trading_date": "2026-06-29", + "stock_official_margin_short_source_runs": [ + { + "source_name": "official_margin_short_daily", + "source_run_id": 3384, + "started_at": "2026-06-29T11:05:17.201062Z", + "status": "official_pending", + "target_date": "2026-06-29" + }, + { + "source_name": "official_margin_short_daily", + "source_run_id": 3383, + "started_at": "2026-06-29T11:05:17.201062Z", + "status": "official_pending", + "target_date": "2026-06-29" + } + ], + "stockplatform_api_healthz_green": true, + "stockplatform_controlled_recovery_gate_required": false, + "stockplatform_controlled_recovery_gate_status": "not_required_yet", + "stockplatform_freshness_endpoint_readback_present": true, + "stockplatform_healthz_green": true, + "stockplatform_ingestion_endpoint_readback_present": true, + "stockplatform_ingestion_status": "blocked", + "stockplatform_scorecard_readback_integrated": true, + "stockplatform_transient_502_before_success_seen": true, + "target_minutes": 10, + "unreachable_hosts": [], + "wazuh_dashboard_degraded": false }, "professional_fix": { - "owner": "reboot auto-recovery lane plus StockPlatform freshness readback", - "action": "Keep the live boot-triggered SLO timer enabled. Do not reboot or restart services from this lane. Treat StockPlatform freshness as the current service recovery blocker: inspect existing ingestion readback, wait the scheduled retry windows, then rerun verify-only; if still blocked after the final retry window, open a controlled data recovery gate instead of writing data manually.", + "action": "Keep the live boot-triggered SLO timer enabled. Do not reboot or restart services from this lane. Treat StockPlatform freshness/ingestion as the current service recovery blocker: use the committed P0-006 scorecard and /api/v1/agents/reboot-auto-recovery-slo-scorecard readback, wait scheduled retry windows, then rerun verify-only. If still blocked after 23:35, open the controlled data recovery gate with source diff, check-mode/dry-run, rollback, and post-freshness verifier; never write manual DB rows or fake freshness.", "exit_criteria": [ "can_claim_all_services_recovered_within_target=true", "observed_hosts=110,120,121,188", @@ -213,61 +292,66 @@ "BACKUP_CORE_GREEN=1", "HOST_188_SERVICE_GREEN=1", "live_slo_metric_present=true" - ] + ], + "owner": "reboot auto-recovery lane plus StockPlatform freshness readback" }, - "safe_next_step": "inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_rerun_slo_verify_only_no_reboot" + "reason": "P0-006 scorecard now directly imports StockPlatform freshness and ingestion readback. At 2026-06-29 before the 23:35 final retry window, StockPlatform public/API health is green but product data remains blocked by core_margin_short_daily_missing, ai_recommendations_stale, and ingestion core.margin_short_daily_incomplete. No reboot, restart, DB write, or fake freshness marker is allowed from this lane; rerun verify-only after retry windows, then open the controlled data recovery gate only if still blocked.", + "safe_next_step": "inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_rerun_slo_verify_only_no_reboot", + "status": "blocked_stockplatform_product_data_freshness_waiting_final_retry_window", + "title": "主機重啟自動偵測、自動觸發與 10 分鐘恢復 SLO", + "workplan_id": "P0-006" }, { - "workplan_id": "P0-005", - "title": "產品資料與備份 contract", - "status": "stock_freshness_blocked_and_waiting_non_secret_credential_escrow_evidence_refs", - "reason": "Backup/offsite core remains green and the credential escrow readiness API correctly exposes safe_next_step. Current product data freshness is blocked after the first EOD window by StockPlatform margin-short official_pending and stale AI recommendations; the separate DR blocker remains the five non-secret escrow evidence refs.", "evidence": { - "product_data_green": false, - "stock_freshness_status": "blocked", - "stock_latest_trading_date": "2026-06-29", "backup_core_green": true, - "dr_escrow_blocked": true, - "summary_escrow_missing_count": 5, - "offsite_configured": true, - "rclone_configured": true, - "script_missing_count": 0, - "credential_marker_write_authorized_count": 0, - "secret_value_collection_allowed": false, "checklist_generator_present": true, - "checklist_schema_version": "awoooi_dr_escrow_evidence_checklist_v1", - "placeholder_preflight_guard_present": true, - "offsite_fresh": true, - "rclone_gdrive_configured": true, - "rclone_gdrive_fresh": true, - "escrow_status_live_checked": true, - "live_no_secret_status_path": "/tmp/awoooi-p0-005-escrow-status-live.txt", "checklist_latest_path": "/tmp/awoooi-dr-escrow-evidence-checklist-current.json", + "checklist_schema_version": "awoooi_dr_escrow_evidence_checklist_v1", "credential_escrow_intake_api_route_live_checked": true, + "credential_marker_write_authorized_count": 0, + "dr_escrow_blocked": true, + "escrow_status_live_checked": true, + "focused_p0_005_tests_passed": 16, + "live_no_secret_status_path": "/tmp/awoooi-p0-005-escrow-status-live.txt", + "offsite_configured": true, + "offsite_fresh": true, + "placeholder_preflight_guard_present": true, + "product_data_green": false, "production_missing_item_count": 5, "production_owner_response_accepted_count": 0, - "production_runtime_gate_count": 0, - "production_secret_value_collection_allowed": false, - "production_top_level_safe_next_step_was_null_before_source_fix": true, - "source_top_level_safe_next_step_present": true, - "source_safe_next_step": "collect_redacted_non_secret_evidence_refs_then_rerun_preflight", - "focused_p0_005_tests_passed": 16, - "production_top_level_safe_next_step_readback": true, - "production_safe_next_step": "collect_redacted_non_secret_evidence_refs_then_rerun_preflight", "production_readback_safe_next_step": "collect_redacted_non_secret_evidence_refs_then_rerun_preflight", + "production_runtime_gate_count": 0, + "production_safe_next_step": "collect_redacted_non_secret_evidence_refs_then_rerun_preflight", + "production_secret_value_collection_allowed": false, + "production_top_level_safe_next_step_readback": true, + "production_top_level_safe_next_step_was_null_before_source_fix": true, + "rclone_configured": true, + "rclone_gdrive_configured": true, + "rclone_gdrive_fresh": true, + "script_missing_count": 0, + "secret_value_collection_allowed": false, + "source_safe_next_step": "collect_redacted_non_secret_evidence_refs_then_rerun_preflight", + "source_top_level_safe_next_step_present": true, "stock_blockers": [ "core_margin_short_daily_missing", "ai_recommendations_stale" ], - "stock_eod_window_pending": false, "stock_eod_classification": "after_first_eod_window_blocked", "stock_eod_next_action": "inspect_ingestion_logs_and_wait_retry_windows", + "stock_eod_window_pending": false, + "stock_freshness_status": "blocked", + "stock_ingestion_blockers": [ + "core.margin_short_daily_incomplete" + ], + "stock_latest_trading_date": "2026-06-29", "stock_official_margin_short_source_runs": [ 3384, 3383 ], + "stockplatform_api_healthz_green": true, "stockplatform_healthz_green": true, - "stockplatform_api_healthz_green": true + "stockplatform_ingestion_status": "blocked", + "summary_escrow_missing_count": 5 }, "missing_items": [ "restic_repository_password", @@ -277,8 +361,11 @@ "oauth_ai_provider_recovery" ], "professional_fix": { - "owner": "DR evidence lane", "action": "Use the single DR escrow checklist packet with the five required item ids, accept only redacted evidence refs, then run the existing preflight once.", + "do_not_repeat": [ + "Do not reopen cold-start or CI/CD work because escrow refs are missing.", + "Do not create additional owner-package variants for the same five refs." + ], "exit_criteria": [ "effective_escrow_missing_count=0", "owner_response_accepted_count=1", @@ -286,35 +373,38 @@ "secret_value_collection_allowed=false", "post_reboot_readiness OVERALL_DECLARATION no longer includes DR_ESCROW_BLOCKED" ], - "do_not_repeat": [ - "Do not reopen cold-start or CI/CD work because escrow refs are missing.", - "Do not create additional owner-package variants for the same five refs." - ] + "owner": "DR evidence lane" }, - "safe_next_step": "first_clear_stockplatform_freshness_through_scheduled_retry_readback_then_collect_five_non_secret_credential_escrow_evidence_refs_and_rerun_single_preflight" + "reason": "Backup/offsite core remains green and the credential escrow readiness API correctly exposes safe_next_step. Current product data freshness is blocked after the first EOD window by StockPlatform margin-short official_pending and stale AI recommendations; the separate DR blocker remains the five non-secret escrow evidence refs.", + "safe_next_step": "first_rerun_p0_006_stockplatform_freshness_after_retry_window_then_collect_five_non_secret_credential_escrow_evidence_refs_and_rerun_single_preflight", + "status": "stock_freshness_blocked_and_waiting_non_secret_credential_escrow_evidence_refs", + "title": "產品資料與備份 contract", + "workplan_id": "P0-005" }, { - "workplan_id": "P0-003", - "title": "取得 Gitea private inventory 權限", - "status": "payload_validator_ready_waiting_authenticated_or_admin_export_inventory", - "reason": "P0-003 now has a Gitea-only scorecard, read-only API, Delivery Workbench active lane, live 110 public inventory refreshed to four repos, and a no-secret machine validator for redacted authenticated/admin export inventory payloads. GitHub is excluded from active P0 blocker math; current public-only inventory should validate as needs_supplement with accepted_payload_count=0, and private/internal coverage still requires an authenticated/admin redacted payload or owner attestation.", "evidence": { - "private_inventory_source": "gitea", - "github_lane_excluded_from_p0_blocker_count": true, + "accepted_inventory_payload_count": 0, + "active_blockers": [ + "gitea_repo_inventory_status_not_ok", + "gitea_visibility_scope_public_only_or_unknown", + "gitea_authenticated_inventory_payload_not_accepted", + "gitea_owner_coverage_attestation_not_received" + ], + "all_active_product_repos_have_gitea_owner_readiness_row": true, "api_readback_schema_version": "gitea_private_inventory_p0_scorecard_readback_v1", + "authenticated_payload_validation_accepted_payload_count": 0, + "authenticated_payload_validation_blocker_count": 4, + "authenticated_payload_validation_snapshot": "docs/operations/awoooi-gitea-authenticated-inventory-payload-validation.snapshot.json", + "authenticated_payload_validation_status": "needs_supplement", + "authenticated_payload_validator_present": true, + "authenticated_payload_validator_source": "scripts/security/gitea-authenticated-inventory-payload-validator.py", + "delivery_workbench_active_blocker_count": 4, "delivery_workbench_active_lane": "gitea_private_inventory", "delivery_workbench_github_blocked_preflight_target_count": 0, "delivery_workbench_review_readiness_percent": 60, - "delivery_workbench_active_blocker_count": 4, - "gitea_inventory_status": "partial", - "gitea_visibility_scope": "public_only", - "gitea_public_repo_count": 4, - "accepted_inventory_payload_count": 0, - "owner_coverage_attestation_received_count": 0, "expected_product_count": 11, - "present_product_row_count": 11, - "missing_product_row_count": 0, - "all_active_product_repos_have_gitea_owner_readiness_row": true, + "gitea_inventory_status": "partial", + "gitea_public_repo_count": 4, "gitea_public_repos": [ "wooo/awoooi", "wooo/ewoooc", @@ -322,26 +412,19 @@ "wooo/2026FIFAWorldCup" ], "gitea_readonly_token_present": false, - "active_blockers": [ - "gitea_repo_inventory_status_not_ok", - "gitea_visibility_scope_public_only_or_unknown", - "gitea_authenticated_inventory_payload_not_accepted", - "gitea_owner_coverage_attestation_not_received" - ], - "authenticated_payload_validation_status": "needs_supplement", - "authenticated_payload_validation_accepted_payload_count": 0, - "authenticated_payload_validation_blocker_count": 4, - "authenticated_payload_validator_present": true, - "authenticated_payload_validator_source": "scripts/security/gitea-authenticated-inventory-payload-validator.py", - "authenticated_payload_validation_snapshot": "docs/operations/awoooi-gitea-authenticated-inventory-payload-validation.snapshot.json", - "token_value_collection_allowed": false, - "repo_write_allowed": false, - "refs_sync_allowed": false, + "gitea_visibility_scope": "public_only", + "github_lane_excluded_from_p0_blocker_count": true, "github_primary_switch_authorized": false, - "runtime_gate_count": 0 + "missing_product_row_count": 0, + "owner_coverage_attestation_received_count": 0, + "present_product_row_count": 11, + "private_inventory_source": "gitea", + "refs_sync_allowed": false, + "repo_write_allowed": false, + "runtime_gate_count": 0, + "token_value_collection_allowed": false }, "professional_fix": { - "owner": "Gitea inventory lane", "action": "Use the Gitea-only P0 scorecard, accept only redacted authenticated/admin export inventory payloads, and keep GitHub as do_not_use historical context only.", "exit_criteria": [ "private_inventory_source=gitea", @@ -349,51 +432,100 @@ "gitea_repo_inventory.status=ok", "gitea_repo_inventory.visibility_scope in authenticated/admin_export", "all_active_product_repos_have_gitea_owner_readiness_row=true" - ] + ], + "owner": "Gitea inventory lane" }, - "safe_next_step": "supply_authenticated_or_admin_export_redacted_inventory_payload_then_run_gitea_authenticated_inventory_payload_validator" + "reason": "P0-003 now has a Gitea-only scorecard, read-only API, Delivery Workbench active lane, live 110 public inventory refreshed to four repos, and a no-secret machine validator for redacted authenticated/admin export inventory payloads. GitHub is excluded from active P0 blocker math; current public-only inventory should validate as needs_supplement with accepted_payload_count=0, and private/internal coverage still requires an authenticated/admin redacted payload or owner attestation.", + "safe_next_step": "supply_authenticated_or_admin_export_redacted_inventory_payload_then_run_gitea_authenticated_inventory_payload_validator", + "status": "payload_validator_ready_waiting_authenticated_or_admin_export_inventory", + "title": "取得 Gitea private inventory 權限", + "workplan_id": "P0-003" } ], + "next_execution_order": [ + "P0-006: StockPlatform freshness/ingestion readback is now integrated into the reboot SLO scorecard; wait the 23:35 final retry window, rerun verify-only with no reboot/restart/DB write, and open controlled data recovery gate only if still blocked.", + "P0-005: after P0-006 freshness clears, collect five non-secret credential escrow evidence refs and rerun one preflight; production API safe_next_step reads back correctly and backup/offsite core remains green.", + "P0-003: supply authenticated/admin redacted inventory payload or owner coverage attestation, then run the Gitea payload validator; public-only live inventory remains four repos and GitHub remains stopped/retired/do_not_use.", + "P1-OPENCLAW-LIVE-OPS-SPACE: build the Gather-like OpenClaw continuous animated live ops workspace only after the current P0 runtime truth blockers are no longer the active next action." + ], + "noise_integrated_risk_register": [ + { + "exit_criteria": "latest_main_deploy_readback_success=true", + "professional_handling": "Only the latest non-canceled run for the current main head is a deploy signal; older canceled runs become context, not blockers.", + "signal": "canceled_or_superseded_cd_runs", + "why_it_matters": "Canceled runs hide whether the latest main actually deployed and can make a completed source fix look failed." + }, + { + "exit_criteria": "production_deploy_status=closure_verified and P0 endpoints unchanged", + "professional_handling": "Keep UI-only changes on controlled-runtime profile and verify production readback after the newest head; do not reopen host recovery.", + "signal": "ui_redaction_commits_during_p0_recovery", + "why_it_matters": "Concurrent UI-only commits can cancel P0 CI runs and consume attention while not changing cold-start or DR truth." + }, + { + "exit_criteria": "all P0 test receipts come from clean main worktree", + "professional_handling": "Run P0 verification in clean Gitea main worktrees; never stage or revert user dirty files during P0 recovery.", + "signal": "dirty_local_awooop_worktree", + "why_it_matters": "Local uncommitted frontend copy can create false test failures and pollute P0 diagnosis." + }, + { + "exit_criteria": "active_p0_next_actions contain no GitHub recovery work", + "professional_handling": "Keep GitHub stopped/do_not_use; remove it from active P0 next actions and route inventory work to Gitea.", + "signal": "legacy_github_lane", + "why_it_matters": "Retired GitHub blockers inflate P0 blocker count and distract from active Gitea/runtime work." + }, + { + "exit_criteria": "cold_start_blocked=0 and warning items assigned outside reboot recovery", + "professional_handling": "Keep warnings visible with owner and due lane; close reboot recovery when BLOCKED=0 and service/data health are green.", + "signal": "warning_only_cold_start_findings", + "why_it_matters": "Warnings can reveal hygiene debt, but treating them as reboot blockers delays recovery closure." + }, + { + "exit_criteria": "priority snapshot current_head matches latest successful deploy marker", + "professional_handling": "Live production readback wins; snapshots must be refreshed only when they change the next executable P0 action.", + "signal": "stale_or_incomplete_priority_snapshots", + "why_it_matters": "Old snapshots can contradict live production readback and send work back to already-fixed items." + }, + { + "exit_criteria": "stockplatform health routes and system freshness/ingestion readback return 200 with freshness status ok", + "professional_handling": "Confirm healthz, api healthz, freshness, and ingestion together; classify runtime down only if repeated health/readback fails, otherwise keep the confirmed blocker on product data freshness.", + "signal": "stockplatform_transient_system_api_502", + "why_it_matters": "A transient 502 on freshness/ingestion can look like service down and can also hide the real data freshness blocker." + } + ], + "operation_boundaries": { + "credential_marker_written": false, + "credential_secret_value_read": false, + "database_write_or_restore_performed": false, + "docker_restart_performed": false, + "firewall_change_performed": false, + "github_api_used": false, + "github_cli_used": false, + "host_write_performed": false, + "k3s_restart_or_node_drain_performed": false, + "nginx_restart_performed": false, + "runner_registration_performed": false, + "secret_or_runner_token_read": false, + "workflow_dispatch_performed": false + }, "post_p0_prioritized_work_items": [ { - "workplan_id": "P1-OPENCLAW-LIVE-OPS-SPACE", - "title": "OpenClaw Gather-like 持續動畫工作室", - "priority": "P1", - "status": "queued_after_current_p0_runtime_truth_blockers", - "reason": "Owner requested an OpenClaw workroom similar to a virtual-office presence app, with a continuous animated scene showing AI Agents working. This is productized AI Agent visibility, not a replacement for current P0 runtime truth closure, so it is queued after P0-006, P0-005, and P0-003 blockers.", + "blocked_operations": [ + "raw_session_read", + "sqlite_read", + "secret_or_token_display", + "GitHub_API_or_gh_usage", + "runtime_write_without_controlled_apply", + "P0_priority_preemption" + ], "evidence": { - "owner_requested_at_local": "2026-06-29", - "reference_pattern": "Gather-like virtual office / presence / walk-up collaboration", "must_be_continuous_animation": true, "must_show_openclaw_working": true, - "static_gif_or_static_card_is_not_sufficient": true, + "owner_requested_at_local": "2026-06-29", "raw_session_or_sqlite_allowed": false, - "secret_or_token_display_allowed": false + "reference_pattern": "Gather-like virtual office / presence / walk-up collaboration", + "secret_or_token_display_allowed": false, + "static_gif_or_static_card_is_not_sufficient": true }, - "scope": { - "experience_name": "OpenClaw Live Ops Space", - "primary_goal": "讓使用者用可視化空間看到 OpenClaw / AI Agent 正在處理任務、等待 verifier、交接、人機協作與回寫狀態。", - "surface_candidates": [ - "/zh-TW/openclaw/live-ops-space", - "/zh-TW/awooop/work-items?view=live-ops" - ], - "core_scene_model": [ - "2D isometric operations room", - "OpenClaw / AI Agent avatars", - "workbench zones for MCP, RAG, KM, PlayBook, verifier, Telegram receipt, deploy/readback", - "presence and attention states", - "task queue and work item movement", - "continuous idle / walking / working / waiting / blocked / verified animations" - ] - }, - "required_integrations_in_order": [ - "Reuse agent-autonomous-runtime-control trace_ledger and work_item_progress as the first read-only event source.", - "Define a public-safe scene_state contract that maps runtime events to avatar state, room zone, task card, and animation state.", - "Add a read-only API loader for OpenClaw live ops scene state; no raw logs, raw sessions, SQLite, token values, or secret payloads.", - "Implement the animated 2D scene with deterministic fallback sample state and live readback state.", - "Add desktop/mobile browser smoke plus animation nonblank checks before production claim.", - "Promote to production only after P0 runtime truth blockers are not the active next action." - ], "exit_criteria": [ "route_renders_nonblank_animated_scene_desktop=true", "route_renders_nonblank_animated_scene_mobile=true", @@ -406,59 +538,36 @@ "browser_console_error_count=0", "production_route_smoke_passed=true" ], - "blocked_operations": [ - "raw_session_read", - "sqlite_read", - "secret_or_token_display", - "GitHub_API_or_gh_usage", - "runtime_write_without_controlled_apply", - "P0_priority_preemption" + "priority": "P1", + "reason": "Owner requested an OpenClaw workroom similar to a virtual-office presence app, with a continuous animated scene showing AI Agents working. This is productized AI Agent visibility, not a replacement for current P0 runtime truth closure, so it is queued after P0-006, P0-005, and P0-003 blockers.", + "required_integrations_in_order": [ + "Reuse agent-autonomous-runtime-control trace_ledger and work_item_progress as the first read-only event source.", + "Define a public-safe scene_state contract that maps runtime events to avatar state, room zone, task card, and animation state.", + "Add a read-only API loader for OpenClaw live ops scene state; no raw logs, raw sessions, SQLite, token values, or secret payloads.", + "Implement the animated 2D scene with deterministic fallback sample state and live readback state.", + "Add desktop/mobile browser smoke plus animation nonblank checks before production claim.", + "Promote to production only after P0 runtime truth blockers are not the active next action." ], - "safe_next_step": "after_current_p0_blockers_stop_being_the_active_next_action_define_openclaw_live_ops_scene_state_contract_and_route_skeleton" - } - ], - "noise_integrated_risk_register": [ - { - "signal": "canceled_or_superseded_cd_runs", - "why_it_matters": "Canceled runs hide whether the latest main actually deployed and can make a completed source fix look failed.", - "professional_handling": "Only the latest non-canceled run for the current main head is a deploy signal; older canceled runs become context, not blockers.", - "exit_criteria": "latest_main_deploy_readback_success=true" - }, - { - "signal": "ui_redaction_commits_during_p0_recovery", - "why_it_matters": "Concurrent UI-only commits can cancel P0 CI runs and consume attention while not changing cold-start or DR truth.", - "professional_handling": "Keep UI-only changes on controlled-runtime profile and verify production readback after the newest head; do not reopen host recovery.", - "exit_criteria": "production_deploy_status=closure_verified and P0 endpoints unchanged" - }, - { - "signal": "dirty_local_awooop_worktree", - "why_it_matters": "Local uncommitted frontend copy can create false test failures and pollute P0 diagnosis.", - "professional_handling": "Run P0 verification in clean Gitea main worktrees; never stage or revert user dirty files during P0 recovery.", - "exit_criteria": "all P0 test receipts come from clean main worktree" - }, - { - "signal": "legacy_github_lane", - "why_it_matters": "Retired GitHub blockers inflate P0 blocker count and distract from active Gitea/runtime work.", - "professional_handling": "Keep GitHub stopped/do_not_use; remove it from active P0 next actions and route inventory work to Gitea.", - "exit_criteria": "active_p0_next_actions contain no GitHub recovery work" - }, - { - "signal": "warning_only_cold_start_findings", - "why_it_matters": "Warnings can reveal hygiene debt, but treating them as reboot blockers delays recovery closure.", - "professional_handling": "Keep warnings visible with owner and due lane; close reboot recovery when BLOCKED=0 and service/data health are green.", - "exit_criteria": "cold_start_blocked=0 and warning items assigned outside reboot recovery" - }, - { - "signal": "stale_or_incomplete_priority_snapshots", - "why_it_matters": "Old snapshots can contradict live production readback and send work back to already-fixed items.", - "professional_handling": "Live production readback wins; snapshots must be refreshed only when they change the next executable P0 action.", - "exit_criteria": "priority snapshot current_head matches latest successful deploy marker" - }, - { - "signal": "stockplatform_transient_system_api_502", - "why_it_matters": "A transient 502 on freshness/ingestion can look like service down and can also hide the real data freshness blocker.", - "professional_handling": "Confirm healthz, api healthz, freshness, and ingestion together; classify runtime down only if repeated health/readback fails, otherwise keep the confirmed blocker on product data freshness.", - "exit_criteria": "stockplatform health routes and system freshness/ingestion readback return 200 with freshness status ok" + "safe_next_step": "after_current_p0_blockers_stop_being_the_active_next_action_define_openclaw_live_ops_scene_state_contract_and_route_skeleton", + "scope": { + "core_scene_model": [ + "2D isometric operations room", + "OpenClaw / AI Agent avatars", + "workbench zones for MCP, RAG, KM, PlayBook, verifier, Telegram receipt, deploy/readback", + "presence and attention states", + "task queue and work item movement", + "continuous idle / walking / working / waiting / blocked / verified animations" + ], + "experience_name": "OpenClaw Live Ops Space", + "primary_goal": "讓使用者用可視化空間看到 OpenClaw / AI Agent 正在處理任務、等待 verifier、交接、人機協作與回寫狀態。", + "surface_candidates": [ + "/zh-TW/openclaw/live-ops-space", + "/zh-TW/awooop/work-items?view=live-ops" + ] + }, + "status": "queued_after_current_p0_runtime_truth_blockers", + "title": "OpenClaw Gather-like 持續動畫工作室", + "workplan_id": "P1-OPENCLAW-LIVE-OPS-SPACE" } ], "professional_execution_rules": [ @@ -468,33 +577,39 @@ "Do not re-run cold-start, CD, or UI checks unless their result can change the next P0 action.", "One blocker gets one checklist and one preflight; do not produce duplicate owner packets for the same missing evidence." ], + "schema_version": "awoooi_priority_work_order_readback_v1", + "source_refs": { + "credential_escrow_intake_readiness_api": "/api/v1/agents/credential-escrow-evidence-intake-readiness", + "credential_escrow_intake_readiness_service": "apps/api/src/services/credential_escrow_evidence_intake_readiness.py", + "credential_escrow_intake_readiness_tests": "apps/api/tests/test_credential_escrow_evidence_intake_readiness_api.py", + "credential_escrow_live_status": "/tmp/awoooi-p0-005-escrow-status-live.txt", + "credential_escrow_scorecard": "/tmp/awoooi-credential-escrow-intake-scorecard-20260629-1200-priority.json", + "delivery_closure_workbench": "https://awoooi.wooo.work/api/v1/agents/delivery-closure-workbench", + "dr_escrow_evidence_checklist_generator": "scripts/reboot-recovery/dr-escrow-evidence-checklist.py", + "dr_escrow_evidence_checklist_latest": "/tmp/awoooi-dr-escrow-evidence-checklist-current.json", + "full_stack_cold_start_check": "scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color", + "gitea_authenticated_inventory_payload_validation_snapshot": "docs/operations/awoooi-gitea-authenticated-inventory-payload-validation.snapshot.json", + "gitea_authenticated_inventory_payload_validator": "scripts/security/gitea-authenticated-inventory-payload-validator.py", + "gitea_private_inventory_p0_scorecard": "docs/operations/awoooi-gitea-private-inventory-p0-scorecard.snapshot.json", + "gitea_repo_inventory_snapshot": "docs/security/gitea-repo-inventory.snapshot.json", + "global_scorecard": "~/.codex/product-runtime-governance-completion-scorecard.snapshot.json", + "owner_requested_openclaw_live_ops_space": "2026-06-29 owner request: Gather-like OpenClaw continuous animated live ops workspace", + "post_reboot_summary": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260629-191950/summary.txt", + "public_gitea_queue_readback": "ops/runner/read-public-gitea-actions-queue.py --json", + "reboot_auto_recovery_slo_metric": "/home/wooo/node_exporter_textfiles/reboot_auto_recovery_slo.prom", + "reboot_auto_recovery_slo_scorecard": "docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json", + "reboot_auto_recovery_slo_scorecard_api": "/api/v1/agents/reboot-auto-recovery-slo-scorecard", + "stockplatform_freshness_readback": "https://stock.wooo.work/api/v1/system/freshness", + "stockplatform_ingestion_readback": "https://stock.wooo.work/api/v1/system/ingestion", + "workstation_dashboard": "~/.codex/codex-workstation-sync-dashboard.snapshot.json" + }, + "status": "p0_ordered_readback_p0_006_stock_freshness_retry_window_waiting_p0_005_refs_waiting_p0_003_payload_validator_ready", "stopped_or_do_not_use": [ { - "workplan_id": "P0-003A", - "title": "GitHub 全產品備援鏡像", + "allowed_actions": 0, "status": "removed_deleted_do_not_use", - "allowed_actions": 0 + "title": "GitHub 全產品備援鏡像", + "workplan_id": "P0-003A" } - ], - "operation_boundaries": { - "github_api_used": false, - "github_cli_used": false, - "workflow_dispatch_performed": false, - "runner_registration_performed": false, - "secret_or_runner_token_read": false, - "credential_secret_value_read": false, - "credential_marker_written": false, - "host_write_performed": false, - "docker_restart_performed": false, - "nginx_restart_performed": false, - "firewall_change_performed": false, - "k3s_restart_or_node_drain_performed": false, - "database_write_or_restore_performed": false - }, - "next_execution_order": [ - "P0-006: keep the live reboot SLO timer active; no reboot or service restart; current blocker is StockPlatform product data freshness after the first EOD window, so inspect ingestion readback, wait retry windows, then rerun verify-only.", - "P0-005: after freshness clears, collect five non-secret credential escrow evidence refs and rerun one preflight; production API safe_next_step reads back correctly and backup/offsite core remains green.", - "P0-003: supply authenticated/admin redacted inventory payload or owner coverage attestation, then run the Gitea payload validator; public-only live inventory remains four repos and GitHub remains stopped/retired/do_not_use.", - "P1-OPENCLAW-LIVE-OPS-SPACE: build the Gather-like OpenClaw continuous animated live ops workspace only after the current P0 runtime truth blockers are no longer the active next action." ] } diff --git a/docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json b/docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json index 7d505aee1..96b7f102d 100644 --- a/docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json +++ b/docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json @@ -3,15 +3,17 @@ "host_boot_observation_older_than_target_window", "post_start_blocked_not_zero", "product_data_green_not_1", - "service_green_not_1" + "service_green_not_1", + "stockplatform_freshness_blocked", + "stockplatform_ingestion_blocked" ], "can_claim_all_services_recovered_within_target": false, "capacity": { "checked": true, - "free_gib": 144.317, + "free_gib": 4.454, "min_free_gib": 2.0 }, - "generated_at": "2026-06-29T19:20:59+08:00", + "generated_at": "2026-06-29T19:49:30+08:00", "host_boot_detection": { "host_rows": [ { @@ -94,7 +96,7 @@ "summary_present": true, "wazuh_dashboard_degraded": false }, - "safe_next_step": "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_rerun_scorecard_until_status_slo_ready", + "safe_next_step": "inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_rerun_slo_verify_only_no_reboot", "schema_version": "awoooi_reboot_auto_recovery_slo_scorecard_v1", "source_controls": { "cold_start_textfile_exporter_source_present": true, @@ -108,6 +110,144 @@ "slo_systemd_timer_source_present": true }, "status": "blocked_reboot_auto_recovery_slo_not_ready", + "stockplatform_data_freshness": { + "blocked_sources": [ + { + "latest_date": "2026-06-26", + "notes": "官方來源狀態:上櫃官方尚未發布,回傳日期 2026-06-29,來源列數 0;上市官方尚未發布,回傳日期 尚無,來源列數 0", + "row_count": 0, + "source": "core.margin_short_daily", + "status": "missing" + }, + { + "latest_date": "2026-06-26", + "notes": "最新日期 2026-06-26 早於最新交易日 2026-06-29", + "row_count": 2988, + "source": "ai.recommendations", + "status": "stale" + } + ], + "controlled_recovery_gate": { + "allowed_actions": [ + "inspect_existing_ingestion_readback", + "prepare_source_of_truth_diff", + "run_check_mode_or_dry_run_import", + "run_post_recovery_freshness_verifier" + ], + "forbidden_actions": [ + "manual_db_update", + "truncate_or_restore", + "fake_freshness_marker", + "secret_value_read", + "reboot_or_service_restart_from_reboot_slo_lane" + ], + "required": false, + "status": "not_required_yet", + "target_selector": "stockplatform-v2:system_freshness:core.margin_short_daily,ai.recommendations" + }, + "eod_window": { + "classification": "after_first_eod_window_blocked", + "final_retry_window_end_local": "23:35", + "final_retry_window_passed": false, + "first_full_window_end_local": "19:15", + "next_action": "inspect_ingestion_logs_and_wait_retry_windows", + "pending": false + }, + "freshness_blockers": [ + "core_margin_short_daily_missing", + "ai_recommendations_stale" + ], + "freshness_endpoint_readback_present": true, + "freshness_status": "blocked", + "ingestion_blockers": [ + "core.margin_short_daily_incomplete" + ], + "ingestion_endpoint_readback_present": true, + "ingestion_status": "blocked", + "latest_source_runs": [ + { + "finished_at": "2026-06-29T11:05:17.201062Z", + "source_name": "official_margin_short_daily", + "source_run_id": 3384, + "started_at": "2026-06-29T11:05:17.201062Z", + "status": "official_pending", + "target_date": "2026-06-29" + }, + { + "finished_at": "2026-06-29T11:05:17.201062Z", + "source_name": "official_margin_short_daily", + "source_run_id": 3383, + "started_at": "2026-06-29T11:05:17.201062Z", + "status": "official_pending", + "target_date": "2026-06-29" + }, + { + "finished_at": "2026-06-29T11:00:11.518268Z", + "source_name": "intelligence_security_linker", + "source_run_id": 3382, + "started_at": "2026-06-29T11:00:11.518268Z", + "status": "succeeded", + "target_date": null + }, + { + "finished_at": "2026-06-29T11:00:10.289509Z", + "source_name": "intelligence_reports_import", + "source_run_id": 3381, + "started_at": "2026-06-29T11:00:10.289509Z", + "status": "succeeded", + "target_date": null + }, + { + "finished_at": "2026-06-29T10:22:15.262488Z", + "source_name": "year_to_date_stock_backfill", + "source_run_id": 3380, + "started_at": "2026-06-29T10:22:15.262488Z", + "status": "planned", + "target_date": null + }, + { + "finished_at": "2026-06-29T10:22:13.464934Z", + "source_name": "official_margin_short_daily", + "source_run_id": 3378, + "started_at": "2026-06-29T10:22:13.464934Z", + "status": "official_pending", + "target_date": "2026-06-29" + }, + { + "finished_at": "2026-06-29T10:22:13.464934Z", + "source_name": "official_margin_short_daily", + "source_run_id": 3379, + "started_at": "2026-06-29T10:22:13.464934Z", + "status": "official_pending", + "target_date": "2026-06-29" + }, + { + "finished_at": "2026-06-29T10:21:56.651793Z", + "source_name": "community_etfinfo_holdings_daily", + "source_run_id": 3345, + "started_at": "2026-06-29T10:21:56.651793Z", + "status": "succeeded", + "target_date": "2026-06-26" + }, + { + "finished_at": "2026-06-29T10:21:56.651793Z", + "source_name": "community_etfinfo_active_changes_daily", + "source_run_id": 3346, + "started_at": "2026-06-29T10:21:56.651793Z", + "status": "succeeded", + "target_date": "2026-06-26" + }, + { + "finished_at": "2026-06-29T10:21:56.651793Z", + "source_name": "community_etfinfo_holdings_daily", + "source_run_id": 3343, + "started_at": "2026-06-29T10:21:56.651793Z", + "status": "succeeded", + "target_date": "2026-06-26" + } + ], + "latest_trading_date": "2026-06-29" + }, "target_minutes": 10, "target_seconds": 600 } diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 7b6393e11..969a75fbd 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -147,6 +147,10 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() - text = _workflow_text() expected_sources = [ "docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json)", + "apps/api/src/services/reboot_auto_recovery_slo_scorecard.py)", + "apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py)", + "src/services/reboot_auto_recovery_slo_scorecard.py", + "tests/test_reboot_auto_recovery_slo_scorecard_api.py", "scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service)", "scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer)", "scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh)", diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index 42c2e0b06..323573446 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -27,6 +27,16 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--target-minutes", type=int, default=10) parser.add_argument("--min-free-gib", type=float, default=2.0) parser.add_argument("--disk-path", type=Path, help="Optionally check local free space.") + parser.add_argument( + "--stock-freshness-file", + type=Path, + help="Optional StockPlatform /api/v1/system/freshness JSON readback.", + ) + parser.add_argument( + "--stock-ingestion-file", + type=Path, + help="Optional StockPlatform /api/v1/system/ingestion JSON readback.", + ) parser.add_argument("--generated-at", help="Override generated_at for stable snapshots.") parser.add_argument("--output", type=Path, help="Write JSON to this path.") return parser.parse_args() @@ -60,6 +70,30 @@ def int_value(value: Any, default: int = 0) -> int: return default +def strings(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if item not in (None, "")] + + +def csv_strings(value: str | None) -> list[str]: + if not value or value in {"none", "unknown"}: + return [] + return [item.strip() for item in value.split(",") if item.strip()] + + +def read_json_object(path: Path | None) -> dict[str, Any]: + if not path: + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {} + if not isinstance(payload, dict): + return {} + return payload + + def parse_host_probe(text: str) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for raw_line in text.splitlines(): @@ -156,10 +190,187 @@ def disk_free_gib(path: Path | None) -> float | None: return int_value(parts[3]) / 1024 / 1024 +def compact_stock_sources(payload: dict[str, Any]) -> list[dict[str, Any]]: + sources = payload.get("sources") + if not isinstance(sources, list): + return [] + rows: list[dict[str, Any]] = [] + for item in sources: + if not isinstance(item, dict): + continue + rows.append( + { + "source": str(item.get("source") or ""), + "latest_date": item.get("latest_date"), + "status": str(item.get("status") or "unknown"), + "row_count": int_value(item.get("row_count")), + "notes": str(item.get("notes") or "")[:500], + } + ) + return rows + + +def compact_source_runs(payload: dict[str, Any]) -> list[dict[str, Any]]: + runs = payload.get("latest_source_runs") + if not isinstance(runs, list): + return [] + rows: list[dict[str, Any]] = [] + for item in runs: + if not isinstance(item, dict): + continue + rows.append( + { + "source_run_id": item.get("source_run_id") or item.get("id"), + "source_name": str(item.get("source_name") or item.get("source") or ""), + "target_date": item.get("target_date") or item.get("date"), + "status": str(item.get("status") or "unknown"), + "started_at": item.get("started_at"), + "finished_at": item.get("finished_at"), + } + ) + return rows + + +def final_retry_window_passed(generated_at: str, final_retry_window_end: str) -> bool: + if not generated_at or not re.match(r"^\d{2}:\d{2}$", final_retry_window_end or ""): + return False + try: + observed_at = datetime.fromisoformat(generated_at) + hour, minute = [int(part) for part in final_retry_window_end.split(":", 1)] + final_at = observed_at.replace(hour=hour, minute=minute, second=0, microsecond=0) + except ValueError: + return False + return observed_at >= final_at + + +def build_stockplatform_readback( + *, + summary: dict[str, str], + freshness: dict[str, Any], + ingestion: dict[str, Any], + generated_at: str, +) -> dict[str, Any]: + freshness_blockers = strings(freshness.get("blockers")) or csv_strings( + summary.get("STOCK_BLOCKERS") + ) + ingestion_blockers = strings(ingestion.get("blockers")) + freshness_status = str( + freshness.get("status") or summary.get("STOCK_FRESHNESS_STATUS") or "unknown" + ) + ingestion_status = str(ingestion.get("status") or "unknown") + latest_trading_date = str( + freshness.get("latest_trading_date") + or ingestion.get("latest_trading_date") + or summary.get("STOCK_LATEST_TRADING_DATE") + or "" + ) + eod_pending = truthy(summary.get("STOCK_EOD_WINDOW_PENDING")) + eod_final_window = str( + summary.get("STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL") or "unknown" + ) + final_passed = final_retry_window_passed(generated_at, eod_final_window) + recovery_required = ( + freshness_status == "blocked" + and not eod_pending + and ( + str(summary.get("STOCK_EOD_NEXT_ACTION") or "") + == "open_stockplatform_data_recovery_gate" + or final_passed + ) + ) + return { + "freshness_endpoint_readback_present": bool(freshness), + "ingestion_endpoint_readback_present": bool(ingestion), + "freshness_status": freshness_status, + "ingestion_status": ingestion_status, + "latest_trading_date": latest_trading_date, + "freshness_blockers": freshness_blockers, + "ingestion_blockers": ingestion_blockers, + "blocked_sources": [ + row + for row in compact_stock_sources(freshness) + if row["status"] not in {"ok", "warning"} + ], + "latest_source_runs": compact_source_runs(ingestion), + "eod_window": { + "pending": eod_pending, + "classification": str(summary.get("STOCK_EOD_CLASSIFICATION") or "unknown"), + "next_action": str(summary.get("STOCK_EOD_NEXT_ACTION") or "unknown"), + "first_full_window_end_local": str( + summary.get("STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL") or "unknown" + ), + "final_retry_window_end_local": eod_final_window, + "final_retry_window_passed": final_passed, + }, + "controlled_recovery_gate": { + "required": recovery_required, + "status": "ready_to_open" if recovery_required else "not_required_yet", + "target_selector": "stockplatform-v2:system_freshness:core.margin_short_daily,ai.recommendations", + "allowed_actions": [ + "inspect_existing_ingestion_readback", + "prepare_source_of_truth_diff", + "run_check_mode_or_dry_run_import", + "run_post_recovery_freshness_verifier", + ], + "forbidden_actions": [ + "manual_db_update", + "truncate_or_restore", + "fake_freshness_marker", + "secret_value_read", + "reboot_or_service_restart_from_reboot_slo_lane", + ], + }, + } + + +def choose_safe_next_step( + *, + blockers: list[str], + stockplatform: dict[str, Any], +) -> str: + freshness_status = str(stockplatform.get("freshness_status") or "unknown") + eod_window = stockplatform.get("eod_window") if isinstance(stockplatform.get("eod_window"), dict) else {} + eod_next_action = str(eod_window.get("next_action") or "") + recovery_gate = ( + stockplatform.get("controlled_recovery_gate") + if isinstance(stockplatform.get("controlled_recovery_gate"), dict) + else {} + ) + if freshness_status == "blocked" and recovery_gate.get("required") is True: + return ( + "open_stockplatform_controlled_data_recovery_gate_with_source_diff_" + "check_mode_rollback_and_post_freshness_verifier_no_manual_db_write" + ) + if freshness_status == "blocked" and eod_next_action in { + "inspect_ingestion_logs_and_wait_retry_windows", + "wait_for_scheduled_eod_window", + }: + return ( + "inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_" + "rerun_slo_verify_only_no_reboot" + ) + if blockers == ["host_boot_observation_older_than_target_window"]: + return ( + "timer_deployed_and_services_readback_green_wait_for_next_all_host_reboot_" + "event_or_approved_reboot_drill_to_prove_10_minute_slo" + ) + return ( + "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_" + "rerun_scorecard_until_status_slo_ready" + ) + + def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: target_seconds = args.target_minutes * 60 + generated_at = args.generated_at or datetime.now().astimezone().isoformat(timespec="seconds") summary = parse_kv(read_text(args.summary_file)) host_rows = parse_host_probe(read_text(args.host_probe_file)) + stockplatform = build_stockplatform_readback( + summary=summary, + freshness=read_json_object(args.stock_freshness_file), + ingestion=read_json_object(args.stock_ingestion_file), + generated_at=generated_at, + ) controls = source_controls() free_gib = disk_free_gib(args.disk_path) @@ -206,6 +417,10 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: blockers.append("service_green_not_1") if not product_data_green: blockers.append("product_data_green_not_1") + if stockplatform["freshness_status"] == "blocked": + blockers.append("stockplatform_freshness_blocked") + if stockplatform["ingestion_status"] == "blocked": + blockers.append("stockplatform_ingestion_blocked") if not backup_core_green: blockers.append("backup_core_green_not_1") if not host_188_service_green: @@ -221,20 +436,13 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: ) unique_blockers = sorted(set(blockers)) can_claim = not unique_blockers - if unique_blockers == ["host_boot_observation_older_than_target_window"]: - safe_next_step = ( - "timer_deployed_and_services_readback_green_wait_for_next_all_host_reboot_" - "event_or_approved_reboot_drill_to_prove_10_minute_slo" - ) - else: - safe_next_step = ( - "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_" - "rerun_scorecard_until_status_slo_ready" - ) + safe_next_step = choose_safe_next_step( + blockers=unique_blockers, + stockplatform=stockplatform, + ) return { "schema_version": SCHEMA_VERSION, - "generated_at": args.generated_at - or datetime.now().astimezone().isoformat(timespec="seconds"), + "generated_at": generated_at, "target_minutes": args.target_minutes, "target_seconds": target_seconds, "status": "slo_ready" if can_claim else "blocked_reboot_auto_recovery_slo_not_ready", @@ -262,6 +470,7 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: "overall_declaration": summary.get("OVERALL_DECLARATION", "unknown"), "next_required_gates": summary.get("NEXT_REQUIRED_GATES", "unknown"), }, + "stockplatform_data_freshness": stockplatform, "capacity": { "checked": free_gib is not None, "free_gib": round(free_gib, 3) if free_gib is not None else None, diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py index cbda41633..2e16442b7 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py @@ -57,6 +57,43 @@ def run_scorecard(tmp_path: Path, summary: str, probe: str = HOST_PROBE_GREEN) - return json.loads(result.stdout) +def run_scorecard_with_stock( + tmp_path: Path, + summary: str, + freshness: dict, + ingestion: dict, + generated_at: str = "2026-06-29T19:41:00+08:00", +) -> dict: + summary_path = tmp_path / "summary.txt" + probe_path = tmp_path / "probe.txt" + freshness_path = tmp_path / "freshness.json" + ingestion_path = tmp_path / "ingestion.json" + summary_path.write_text(summary, encoding="utf-8") + probe_path.write_text(HOST_PROBE_GREEN, encoding="utf-8") + freshness_path.write_text(json.dumps(freshness), encoding="utf-8") + ingestion_path.write_text(json.dumps(ingestion), encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--summary-file", + str(summary_path), + "--host-probe-file", + str(probe_path), + "--stock-freshness-file", + str(freshness_path), + "--stock-ingestion-file", + str(ingestion_path), + "--generated-at", + generated_at, + ], + text=True, + capture_output=True, + check=True, + ) + return json.loads(result.stdout) + + def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -> None: payload = run_scorecard(tmp_path, GREEN_SUMMARY) @@ -99,3 +136,97 @@ def test_services_green_but_old_boot_window_waits_for_reboot_event(tmp_path: Pat "timer_deployed_and_services_readback_green_wait_for_next_all_host_reboot_" "event_or_approved_reboot_drill_to_prove_10_minute_slo" ) + + +def test_stockplatform_blocked_before_final_retry_waits_for_readback(tmp_path: Path) -> None: + summary = GREEN_SUMMARY.replace("PRODUCT_DATA_GREEN=1", "PRODUCT_DATA_GREEN=0").replace( + "SERVICE_GREEN=1", "SERVICE_GREEN=0" + ) + """\ +STOCK_FRESHNESS_STATUS=blocked +STOCK_LATEST_TRADING_DATE=2026-06-29 +STOCK_BLOCKERS=core_margin_short_daily_missing,ai_recommendations_stale +STOCK_EOD_WINDOW_PENDING=0 +STOCK_EOD_CLASSIFICATION=after_first_eod_window_blocked +STOCK_EOD_NEXT_ACTION=inspect_ingestion_logs_and_wait_retry_windows +STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL=19:15 +STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL=23:35 +""" + freshness = { + "status": "blocked", + "latest_trading_date": "2026-06-29", + "blockers": ["core_margin_short_daily_missing", "ai_recommendations_stale"], + "sources": [ + { + "source": "core.margin_short_daily", + "latest_date": "2026-06-26", + "status": "missing", + "row_count": 0, + "notes": "official source pending", + } + ], + } + ingestion = { + "status": "blocked", + "latest_trading_date": "2026-06-29", + "blockers": ["core.margin_short_daily_incomplete"], + "latest_source_runs": [ + { + "source_run_id": 3384, + "source_name": "official_margin_short_daily", + "target_date": "2026-06-29", + "status": "official_pending", + } + ], + } + + payload = run_scorecard_with_stock(tmp_path, summary, freshness, ingestion) + + assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready" + assert "stockplatform_freshness_blocked" in payload["active_blockers"] + assert "stockplatform_ingestion_blocked" in payload["active_blockers"] + assert payload["stockplatform_data_freshness"]["freshness_blockers"] == [ + "core_margin_short_daily_missing", + "ai_recommendations_stale", + ] + assert payload["stockplatform_data_freshness"]["eod_window"][ + "final_retry_window_passed" + ] is False + assert payload["stockplatform_data_freshness"]["controlled_recovery_gate"][ + "required" + ] is False + assert payload["safe_next_step"] == ( + "inspect_stockplatform_ingestion_readback_and_wait_retry_windows_then_" + "rerun_slo_verify_only_no_reboot" + ) + + +def test_stockplatform_blocked_after_final_retry_opens_controlled_gate( + tmp_path: Path, +) -> None: + summary = GREEN_SUMMARY.replace("PRODUCT_DATA_GREEN=1", "PRODUCT_DATA_GREEN=0").replace( + "SERVICE_GREEN=1", "SERVICE_GREEN=0" + ) + """\ +STOCK_FRESHNESS_STATUS=blocked +STOCK_BLOCKERS=core_margin_short_daily_missing +STOCK_EOD_WINDOW_PENDING=0 +STOCK_EOD_CLASSIFICATION=after_final_retry_window_blocked +STOCK_EOD_NEXT_ACTION=open_stockplatform_data_recovery_gate +STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL=23:35 +""" + + payload = run_scorecard_with_stock( + tmp_path, + summary, + {"status": "blocked", "blockers": ["core_margin_short_daily_missing"]}, + {"status": "blocked", "blockers": ["core.margin_short_daily_incomplete"]}, + generated_at="2026-06-29T23:40:00+08:00", + ) + + gate = payload["stockplatform_data_freshness"]["controlled_recovery_gate"] + assert gate["required"] is True + assert gate["status"] == "ready_to_open" + assert "manual_db_update" in gate["forbidden_actions"] + assert payload["safe_next_step"] == ( + "open_stockplatform_controlled_data_recovery_gate_with_source_diff_" + "check_mode_rollback_and_post_freshness_verifier_no_manual_db_write" + )