feat(api): expose stockplatform controlled recovery preflight
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled

This commit is contained in:
Your Name
2026-06-30 16:53:25 +08:00
parent e12a8a4317
commit adc0d8816e
8 changed files with 1264 additions and 0 deletions

View File

@@ -336,6 +336,8 @@ jobs:
;;
apps/api/src/services/stockplatform_public_api_runtime_readback.py)
;;
apps/api/src/services/stockplatform_public_api_controlled_recovery_preflight.py)
;;
apps/api/src/services/iwooos_security_operating_system.py)
;;
apps/api/Dockerfile)
@@ -440,6 +442,8 @@ jobs:
;;
apps/api/tests/test_stockplatform_public_api_runtime_readback.py)
;;
apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py)
;;
apps/api/tests/test_iwooos_security_operating_system.py)
;;
apps/api/tests/test_iwooos_wazuh_prod_manifest.py)
@@ -635,6 +639,7 @@ jobs:
src/services/reboot_auto_recovery_slo_scorecard.py \
src/services/reboot_auto_recovery_drill_preflight.py \
src/services/stockplatform_public_api_runtime_readback.py \
src/services/stockplatform_public_api_controlled_recovery_preflight.py \
src/services/iwooos_security_operating_system.py \
src/services/awoooi_gitea_onboarding_warning_step_dashboard.py \
src/services/awoooi_gitea_onboarding_warning_step_owner_package.py \
@@ -693,6 +698,7 @@ jobs:
tests/test_gitea_workflow_runner_owner_attestation_request_api.py \
tests/test_reboot_auto_recovery_slo_scorecard_api.py \
tests/test_stockplatform_public_api_runtime_readback.py \
tests/test_stockplatform_public_api_controlled_recovery_preflight.py \
tests/test_iwooos_security_operating_system.py \
tests/test_awoooi_production_deploy_readback_blocker.py \
tests/test_awoooi_priority_work_order_readback_api.py \

View File

@@ -335,6 +335,7 @@ from src.services.awoooi_gitea_onboarding_warning_step_template_copy_receipt imp
load_latest_awoooi_gitea_onboarding_warning_step_template_copy_receipt,
)
from src.services.awoooi_priority_work_order_readback import (
apply_stockplatform_public_api_controlled_recovery_preflight,
apply_stockplatform_public_api_runtime_readback,
load_latest_awoooi_priority_work_order_readback,
)
@@ -443,6 +444,9 @@ from src.services.service_health_failure_notification_policy import (
from src.services.service_health_gap_matrix import (
load_latest_service_health_gap_matrix,
)
from src.services.stockplatform_public_api_controlled_recovery_preflight import (
load_latest_stockplatform_public_api_controlled_recovery_preflight,
)
from src.services.stockplatform_public_api_runtime_readback import (
load_latest_stockplatform_public_api_runtime_readback,
)
@@ -1086,6 +1090,14 @@ async def get_awoooi_priority_work_order_readback() -> dict[str, Any]:
load_latest_stockplatform_public_api_runtime_readback
)
apply_stockplatform_public_api_runtime_readback(payload, stockplatform_runtime)
stockplatform_recovery = await asyncio.to_thread(
load_latest_stockplatform_public_api_controlled_recovery_preflight,
runtime_readback=stockplatform_runtime,
)
apply_stockplatform_public_api_controlled_recovery_preflight(
payload,
stockplatform_recovery,
)
return redact_public_lan_topology(payload)
except FileNotFoundError as exc:
raise HTTPException(
@@ -1128,6 +1140,36 @@ async def get_stockplatform_public_api_runtime_readback() -> dict[str, Any]:
) from exc
@router.get(
"/stockplatform-public-api-controlled-recovery-preflight",
response_model=dict[str, Any],
summary="取得 StockPlatform public API controlled recovery preflight",
description=(
"讀取 StockPlatform public API runtime drift並產生 AI controlled "
"recovery preflighttarget selector、source-of-truth diff、bounded "
"action candidates、rollback/verifier 與 KM/RAG/MCP/PlayBook writeback "
"contract。此端點不 SSH、不 Docker、不 restart、不寫 DB、不讀 secret、"
"不觸發 workflow。"
),
)
async def get_stockplatform_public_api_controlled_recovery_preflight() -> dict[str, Any]:
"""回傳 StockPlatform public API controlled recovery preflight。"""
try:
payload = await asyncio.to_thread(
load_latest_stockplatform_public_api_controlled_recovery_preflight
)
return redact_public_lan_topology(payload)
except (json.JSONDecodeError, ValueError) as exc:
logger.error(
"stockplatform_public_api_controlled_recovery_preflight_invalid",
error=str(exc),
)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="StockPlatform public API controlled recovery preflight 無效",
) from exc
@router.get(
"/product-awoooi-manifest-standard",
response_model=dict[str, Any],

View File

@@ -184,6 +184,247 @@ def apply_stockplatform_public_api_runtime_readback(
_refresh_rollups_after_stockplatform_overlay(payload, state)
def apply_stockplatform_public_api_controlled_recovery_preflight(
payload: dict[str, Any],
recovery_preflight: dict[str, Any],
) -> None:
"""Overlay the packaged StockPlatform controlled recovery preflight."""
status = str(recovery_preflight.get("status") or "unknown")
if status == "stockplatform_public_api_recovery_preflight_not_required":
return
state = _dict(payload.setdefault("mainline_execution_state", {}))
rollups = _dict(recovery_preflight.get("rollups"))
source_diff = _dict(recovery_preflight.get("source_of_truth_diff"))
target_selector = _dict(recovery_preflight.get("target_selector"))
controlled_policy = _dict(recovery_preflight.get("controlled_apply_policy"))
candidate_packaged = bool(
rollups.get("controlled_recovery_candidate_packaged") is True
)
api_layer = str(
recovery_preflight.get("api_layer_classification") or "unknown"
)
state["stockplatform_public_api_controlled_recovery_preflight_status"] = status
state["stockplatform_public_api_controlled_recovery_api_layer"] = api_layer
state["stockplatform_public_api_controlled_recovery_active_blockers"] = _strings(
recovery_preflight.get("active_blockers")
)
state["active_p0_state"] = (
"blocked_stockplatform_public_api_controlled_recovery_preflight"
)
state["next_executable_mainline_workplan_id"] = (
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT"
)
state["next_executable_mainline_state"] = _stockplatform_recovery_next_state(
candidate_packaged=candidate_packaged,
api_layer=api_layer,
)
existing_blockers = _strings(state.get("active_p0_live_active_blockers"))
state["active_p0_live_active_blockers"] = _unique_strings(
existing_blockers
+ ["stockplatform_public_api_controlled_recovery_preflight"]
+ _strings(recovery_preflight.get("active_blockers"))
)
for item in _list(payload.get("in_progress_or_blocked_in_priority_order")):
workplan = _dict(item)
if workplan.get("workplan_id") != "P0-006":
continue
evidence = _dict(workplan.setdefault("evidence", {}))
evidence["stockplatform_controlled_recovery_preflight_status"] = status
evidence["stockplatform_controlled_recovery_api_layer"] = state[
"stockplatform_public_api_controlled_recovery_api_layer"
]
evidence["stockplatform_route_target_ready"] = bool(
target_selector.get("route_target_ready") is True
)
evidence["stockplatform_source_of_truth_diff_status"] = str(
source_diff.get("status") or ""
)
evidence["stockplatform_controlled_recovery_candidate_packaged"] = (
candidate_packaged
)
evidence["stockplatform_host_control_path_readback_required"] = bool(
rollups.get("host_control_path_readback_required") is True
)
evidence["stockplatform_controlled_work_item_count"] = _int(
rollups.get("controlled_work_item_count")
)
evidence["stockplatform_post_apply_verifier_count"] = _int(
rollups.get("post_apply_verifier_count")
)
evidence["stockplatform_learning_writeback_contract_count"] = _int(
rollups.get("learning_writeback_contract_count")
)
evidence["stockplatform_current_apply_allowed"] = bool(
controlled_policy.get("current_apply_allowed") is True
)
evidence["stockplatform_runtime_write_gate"] = str(
controlled_policy.get("runtime_write_gate") or ""
)
workplan["status"] = (
"blocked_stockplatform_public_api_controlled_recovery_preflight"
)
workplan["safe_next_step"] = str(
recovery_preflight.get("safe_next_step") or ""
)
workplan["reason"] = (
_stockplatform_recovery_reason(
candidate_packaged=candidate_packaged,
api_layer=api_layer,
)
)
professional_fix = _dict(workplan.setdefault("professional_fix", {}))
professional_fix["action"] = _stockplatform_recovery_action(
api_layer=api_layer,
)
professional_fix["owner"] = (
"StockPlatform API controlled recovery preflight plus P0-006 "
"priority readback"
)
payload["status"] = (
"p0_006_blocked_stockplatform_public_api_controlled_recovery_preflight"
)
payload["next_execution_order"] = [
_stockplatform_recovery_first_order(api_layer=api_layer),
(
"P0-006-STOCKPLATFORM-PUBLIC-API-RUNTIME-READBACK: rerun public "
"health/freshness/ingestion after the bounded reconcile candidate; "
"public API must be green before returning to reboot-window proof."
),
(
"P0-006: keep reboot SLO timer live, but do not claim final SLO "
"closure while StockPlatform public API readback is blocked."
),
(
"NEXT: after StockPlatform public API readback is green or a "
"separate hard-blocker handoff is recorded, resume the next "
"blocker-free priority item."
),
]
_refresh_rollups_after_stockplatform_overlay(payload, state)
summary = _dict(payload.setdefault("summary", {}))
summary["stockplatform_public_api_controlled_recovery_preflight_status"] = status
summary["stockplatform_public_api_controlled_recovery_api_layer"] = str(
recovery_preflight.get("api_layer_classification") or "unknown"
)
summary["stockplatform_public_api_controlled_recovery_candidate_packaged"] = bool(
candidate_packaged
)
summary["stockplatform_public_api_host_control_path_readback_required"] = bool(
rollups.get("host_control_path_readback_required") is True
)
def _stockplatform_recovery_next_state(
*,
candidate_packaged: bool,
api_layer: str,
) -> str:
if api_layer == "api_live_data_dependency_not_ready":
return (
"controlled_recovery_preflight_waiting_stockplatform_data_dependency_"
"readback_and_source_freshness_contract"
)
if candidate_packaged:
return (
"controlled_recovery_packaged_waiting_stockplatform_110_control_path_"
"readback_and_read_only_container_inspection"
)
if api_layer == "public_edge_or_route_unavailable":
return (
"controlled_recovery_preflight_waiting_public_edge_or_188_to_110_"
"upstream_reprobe_before_targeted_apply"
)
return "controlled_recovery_preflight_waiting_route_or_runtime_evidence"
def _stockplatform_recovery_reason(
*,
candidate_packaged: bool,
api_layer: str,
) -> str:
if api_layer == "api_live_data_dependency_not_ready":
return (
"Live StockPlatform API health is reachable, but freshness and/or "
"ingestion readbacks are not ok. The P0-006 lane now has a "
"controlled data-dependency preflight with target selector, "
"source-of-truth diff, data dependency work items, post-apply "
"verifiers, and KM/RAG/MCP/PlayBook writeback contracts. The "
"remaining blocker is data dependency/source freshness control-path "
"readback, not a manual end state."
)
if candidate_packaged:
return (
"Live StockPlatform public web is reachable but public API routes "
"return 502. The P0-006 lane now has a controlled recovery package "
"with target selector, source-of-truth diff, bounded action "
"candidates, post-apply verifiers, and KM/RAG/MCP/PlayBook "
"writeback contracts. The remaining blocker is the StockPlatform "
"110 control-path readback, not a manual end state."
)
if api_layer == "public_edge_or_route_unavailable":
return (
"Live StockPlatform public API recovery preflight is active, but "
"the current public probe cannot prove the edge-alive/API-upstream "
"502 shape. The next AI-controlled step is a bounded public-edge "
"and 188-to-110 upstream reprobe before any targeted apply; this is "
"not a manual end state."
)
return (
"StockPlatform public API recovery preflight is active and waiting for "
"route/runtime evidence before any bounded apply; this is not a manual "
"end state."
)
def _stockplatform_recovery_action(*, api_layer: str) -> str:
if api_layer == "api_live_data_dependency_not_ready":
return (
"Run the StockPlatform data dependency controlled preflight in order: "
"restore/control the 110 host readback path, inspect freshness, "
"ingestion, source freshness and Postgres contract status in read-only "
"mode, apply only metadata/config contract repair after dry-run if "
"needed, then rerun public health/freshness/ingestion and write "
"metadata-only KM/RAG/MCP/PlayBook receipts. Do not restart Docker "
"daemon, reboot hosts, reload Nginx, write DB rows, fake freshness, "
"trigger workflows, or read secrets from this lane."
)
return (
"Run the StockPlatform API controlled recovery preflight in order: "
"restore/control the 110 host readback path, perform read-only API "
"container inspection with redacted logs, reconcile only the API "
"container if inspection proves it is missing/exited/unhealthy, then "
"rerun public health/freshness/ingestion and write metadata-only "
"KM/RAG/MCP/PlayBook receipts. Do not restart Docker daemon, reboot "
"hosts, reload Nginx, write DB rows, trigger workflows, or read "
"secrets from this lane."
)
def _stockplatform_recovery_first_order(*, api_layer: str) -> str:
if api_layer == "api_live_data_dependency_not_ready":
return (
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT: "
"StockPlatform API health is reachable, so the active mainline item "
"is data dependency/source freshness control-path readback, "
"read-only contract inspection, verifier, and metadata writeback; "
"do not DB-write or fake freshness from this lane."
)
return (
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT: "
"controlled recovery package is now the active mainline item; "
"restore/control the 110 readback path, inspect the API container, "
"and only then allow a targeted API reconcile with verifier and "
"metadata writeback."
)
def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
from src.services.awoooi_gitea_onboarding_warning_step_runtime_enablement_gate import (
load_latest_awoooi_gitea_onboarding_warning_step_runtime_enablement_gate,

View File

@@ -0,0 +1,614 @@
"""StockPlatform public API controlled recovery preflight.
This module packages the next P0-006 action after public API 502 drift is
detected. It does not SSH, run Docker, restart services, write databases, read
secrets, or trigger workflows.
"""
from __future__ import annotations
import re
from pathlib import Path
from typing import Any
from src.services.stockplatform_public_api_runtime_readback import (
load_latest_stockplatform_public_api_runtime_readback,
)
_SCHEMA_VERSION = "stockplatform_public_api_controlled_recovery_preflight_v1"
_DEFAULT_ROUTE_SOURCE = (
Path(__file__).resolve().parents[4]
/ "infra"
/ "ansible"
/ "roles"
/ "nginx"
/ "templates"
/ "188-all-sites.conf.j2"
)
def load_latest_stockplatform_public_api_controlled_recovery_preflight(
*,
runtime_readback: dict[str, Any] | None = None,
route_source_path: Path | None = None,
) -> dict[str, Any]:
"""Build the controlled recovery preflight for the active P0-006 drift."""
runtime = runtime_readback or load_latest_stockplatform_public_api_runtime_readback()
route_contract = _load_route_contract(route_source_path or _DEFAULT_ROUTE_SOURCE)
return _build_payload(runtime=runtime, route_contract=route_contract)
def _build_payload(
*,
runtime: dict[str, Any],
route_contract: dict[str, Any],
) -> dict[str, Any]:
runtime_ready = runtime.get("status") == "stockplatform_public_api_runtime_ready"
readback = _dict(runtime.get("readback"))
rollups = _dict(runtime.get("rollups"))
api_layer_classification = _classify_api_layer(readback, rollups, runtime_ready)
route_target_ready = route_contract.get("route_target_ready") is True
recovery_required = not runtime_ready
candidate_packaged = (
recovery_required
and api_layer_classification
in {
"edge_alive_api_upstream_unavailable",
"api_live_data_dependency_not_ready",
}
and route_target_ready
)
active_blockers = _active_blockers(
runtime=runtime,
route_contract=route_contract,
candidate_packaged=candidate_packaged,
runtime_ready=runtime_ready,
)
status = _status(
runtime_ready=runtime_ready,
candidate_packaged=candidate_packaged,
route_target_ready=route_target_ready,
api_layer_classification=api_layer_classification,
)
return {
"schema_version": _SCHEMA_VERSION,
"priority": "P0-006",
"scope": "stockplatform_public_api_controlled_recovery_preflight",
"status": status,
"safe_next_step": _safe_next_step(
runtime_ready=runtime_ready,
candidate_packaged=candidate_packaged,
route_target_ready=route_target_ready,
api_layer_classification=api_layer_classification,
),
"active_blockers": active_blockers,
"active_blocker_count": len(active_blockers),
"api_layer_classification": api_layer_classification,
"runtime_readback_status": str(runtime.get("status") or "unknown"),
"target_selector": {
"product_id": "stockplatform-v2",
"public_domain": "stock.wooo.work",
"public_base_url": str(runtime.get("base_url") or ""),
"edge_route_owner": "188 nginx public ingress source-of-truth",
"runtime_host_role": "stockplatform container runtime host",
"route_source_file": str(route_contract.get("route_source_file") or ""),
"route_target": str(route_contract.get("route_target") or ""),
"route_target_ready": route_target_ready,
"required_public_endpoints": [
"/healthz",
"/api/healthz",
"/api/v1/system/freshness",
"/api/v1/system/ingestion",
],
},
"source_of_truth_diff": {
"status": _source_of_truth_diff_status(
runtime_ready=runtime_ready,
route_target_ready=route_target_ready,
api_layer_classification=api_layer_classification,
),
"public_web_health_http_status": readback.get("web_health_http_status"),
"public_api_health_http_status": readback.get("api_health_http_status"),
"freshness_http_status": readback.get("freshness_http_status"),
"ingestion_http_status": readback.get("ingestion_http_status"),
"http_502_count": _int(rollups.get("http_502_count")),
"route_source_file": str(route_contract.get("route_source_file") or ""),
"route_target": str(route_contract.get("route_target") or ""),
},
"controlled_apply_policy": {
"risk_level": "high",
"break_glass_required": False,
"runtime_write_gate": "controlled_after_preflight",
"controlled_recovery_candidate_packaged": candidate_packaged,
"current_apply_allowed": False,
"current_apply_blocker": _current_apply_blocker(
runtime_ready=runtime_ready,
candidate_packaged=candidate_packaged,
api_layer_classification=api_layer_classification,
),
"allowed_scope": _allowed_scope(api_layer_classification),
"forbidden_scope": [
"docker_daemon_restart",
"host_reboot",
"nginx_reload_or_restart",
"database_write_restore_or_prune",
"secret_value_read",
"workflow_dispatch",
"force_push_or_repo_ref_mutation",
],
},
"controlled_work_items": _controlled_work_items(
candidate_packaged=candidate_packaged,
runtime_ready=runtime_ready,
api_layer_classification=api_layer_classification,
),
"controlled_action_candidates": _controlled_action_candidates(
candidate_packaged=candidate_packaged,
runtime_ready=runtime_ready,
api_layer_classification=api_layer_classification,
),
"post_apply_verifiers": _post_apply_verifiers(runtime_ready=runtime_ready),
"learning_writeback_contracts": _learning_writeback_contracts(),
"rollups": {
"runtime_ready": runtime_ready,
"recovery_required": recovery_required,
"route_target_ready": route_target_ready,
"controlled_recovery_candidate_packaged": candidate_packaged,
"controlled_recovery_preflight_ready": (
runtime_ready or (candidate_packaged and not active_blockers)
),
"host_control_path_readback_required": (
candidate_packaged and not runtime_ready
),
"controlled_work_item_count": len(
_controlled_work_items(
candidate_packaged=candidate_packaged,
runtime_ready=runtime_ready,
api_layer_classification=api_layer_classification,
)
),
"post_apply_verifier_count": len(
_post_apply_verifiers(runtime_ready=runtime_ready)
),
"learning_writeback_contract_count": len(_learning_writeback_contracts()),
},
"operation_boundaries": {
"read_only_public_https_probe": True,
"source_route_file_read": True,
"ssh_used": False,
"docker_command_performed": False,
"docker_restart_performed": False,
"docker_daemon_restart_performed": False,
"host_reboot_performed": False,
"nginx_reload_or_restart_performed": False,
"database_write_or_restore_performed": False,
"stockplatform_manual_data_write_performed": False,
"secret_value_collection_allowed": False,
"github_api_used": False,
"workflow_trigger_performed": False,
"runtime_write_allowed_by_this_readback": False,
},
}
def _load_route_contract(path: Path) -> dict[str, Any]:
payload: dict[str, Any] = {
"route_source_file": str(path),
"route_target": "",
"route_target_ready": False,
"route_source_present": path.exists(),
}
if not path.exists():
return payload
text = path.read_text(encoding="utf-8")
stock_section = _stock_section(text)
targets = re.findall(r"proxy_pass\s+(http://[^;]+);", stock_section)
unique_targets = _unique_strings(targets)
payload["route_target"] = unique_targets[0] if unique_targets else ""
payload["route_targets"] = unique_targets
payload["route_target_ready"] = bool(
"server_name stock.wooo.work;" in stock_section and unique_targets
)
return payload
def _stock_section(text: str) -> str:
start = text.find("# Stock Platform")
if start == -1:
return text
next_marker = text.find("\n# =============================================================================\n# ", start + 80)
if next_marker == -1:
return text[start:]
return text[start:next_marker]
def _classify_api_layer(
readback: dict[str, Any],
rollups: dict[str, Any],
runtime_ready: bool,
) -> str:
if runtime_ready:
return "public_api_ready"
if (
readback.get("web_health_http_status") == 200
and readback.get("api_health_http_status") == 502
and _int(rollups.get("http_502_count")) >= 1
):
return "edge_alive_api_upstream_unavailable"
if readback.get("api_health_http_status") == 200 and (
readback.get("freshness_http_status") == 200
or readback.get("ingestion_http_status") == 200
):
return "api_live_data_dependency_not_ready"
return "public_edge_or_route_unavailable"
def _source_of_truth_diff_status(
*,
runtime_ready: bool,
route_target_ready: bool,
api_layer_classification: str,
) -> str:
if runtime_ready:
return "no_runtime_recovery_diff_required"
if not route_target_ready:
return "route_source_target_missing"
if api_layer_classification == "edge_alive_api_upstream_unavailable":
return "route_source_present_runtime_api_upstream_blocked"
if api_layer_classification == "api_live_data_dependency_not_ready":
return "route_source_present_api_live_data_dependency_not_ready"
return "route_source_present_public_edge_or_route_reprobe_required"
def _active_blockers(
*,
runtime: dict[str, Any],
route_contract: dict[str, Any],
candidate_packaged: bool,
runtime_ready: bool,
) -> list[str]:
if runtime_ready:
return []
blockers = _strings(runtime.get("active_blockers"))
if route_contract.get("route_target_ready") is not True:
blockers.append("stockplatform_route_target_selector_missing")
if candidate_packaged:
blockers.append("stockplatform_110_control_path_readback_required")
return _unique_strings(blockers)
def _current_apply_blocker(
*,
runtime_ready: bool,
candidate_packaged: bool,
api_layer_classification: str,
) -> str:
if runtime_ready:
return ""
if not candidate_packaged:
return "route_or_public_edge_evidence_required"
if api_layer_classification == "api_live_data_dependency_not_ready":
return (
"data_dependency_control_path_readback_required_before_source_"
"freshness_or_postgres_contract_apply"
)
return "host_control_path_readback_required_before_targeted_api_reconcile"
def _allowed_scope(api_layer_classification: str) -> list[str]:
common = [
"public_api_post_apply_verifier",
"km_rag_mcp_playbook_metadata_writeback",
]
if api_layer_classification == "api_live_data_dependency_not_ready":
return [
"read_only_data_dependency_status_inspection",
"read_only_source_freshness_contract_diff",
"read_only_postgres_connection_contract_readback",
"controlled_metadata_or_config_contract_apply_after_dry_run",
*common,
]
return [
"read_only_container_status_inspection",
"read_only_redacted_api_log_tail",
"targeted_api_container_reconcile_after_failed_health_check",
*common,
]
def _status(
*,
runtime_ready: bool,
candidate_packaged: bool,
route_target_ready: bool,
api_layer_classification: str,
) -> str:
if runtime_ready:
return "stockplatform_public_api_recovery_preflight_not_required"
if (
candidate_packaged
and api_layer_classification == "api_live_data_dependency_not_ready"
):
return "controlled_data_dependency_preflight_packaged_waiting_control_path_readback"
if candidate_packaged:
return "controlled_recovery_preflight_packaged_waiting_control_path_readback"
if route_target_ready:
return "blocked_public_edge_or_route_symptom_requires_reprobe"
return "blocked_route_target_selector_missing"
def _safe_next_step(
*,
runtime_ready: bool,
candidate_packaged: bool,
route_target_ready: bool,
api_layer_classification: str,
) -> str:
if runtime_ready:
return "stockplatform_public_api_ready_keep_p0_006_reboot_window_gate"
if (
candidate_packaged
and api_layer_classification == "api_live_data_dependency_not_ready"
):
return (
"run_stockplatform_data_dependency_readback_then_source_freshness_"
"and_postgres_contract_preflight_no_db_write"
)
if candidate_packaged:
return (
"restore_stockplatform_110_control_path_readback_then_run_read_only_"
"container_inspection_and_targeted_api_reconcile_with_public_verifier"
)
if route_target_ready:
return "rerun_public_api_readback_and_188_to_110_upstream_probe_before_apply"
return "repair_source_route_target_selector_before_any_runtime_recovery"
def _controlled_work_items(
*,
candidate_packaged: bool,
runtime_ready: bool,
api_layer_classification: str,
) -> list[dict[str, Any]]:
if runtime_ready:
return []
base_status = "queued_waiting_control_path_readback" if candidate_packaged else "blocked_waiting_route_evidence"
if api_layer_classification == "api_live_data_dependency_not_ready":
return [
{
"id": "stockplatform-data-control-path-readback",
"title": "StockPlatform data dependency control path readback",
"owner_agent": "DevOps Agent",
"status": base_status,
"required_output": "non_secret_host_liveness_and_runtime_contract_match",
},
{
"id": "stockplatform-data-dependency-inspection",
"title": "Read-only freshness / ingestion dependency inspection",
"owner_agent": "SRE Agent",
"status": "waiting_control_path_readback",
"required_output": "freshness_ingestion_source_and_postgres_contract_status",
},
{
"id": "stockplatform-data-contract-preflight",
"title": "Source freshness and Postgres contract preflight",
"owner_agent": "Data Reliability Agent",
"status": "waiting_read_only_inspection",
"required_output": "dry_run_contract_diff_no_db_write",
},
{
"id": "stockplatform-api-post-apply-verifier",
"title": "Public API post-apply verifier",
"owner_agent": "Verifier Agent",
"status": "ready",
"required_output": "public_health_freshness_ingestion_green",
},
{
"id": "stockplatform-data-learning-writeback",
"title": "KM / RAG / MCP / PlayBook trust writeback",
"owner_agent": "Learning Agent",
"status": "ready",
"required_output": "metadata_only_data_dependency_receipt_and_trust_delta",
},
]
return [
{
"id": "stockplatform-api-control-path-readback",
"title": "StockPlatform host control path readback",
"owner_agent": "DevOps Agent",
"status": base_status,
"required_output": "non_secret_host_liveness_and_compose_project_match",
},
{
"id": "stockplatform-api-runtime-inspection",
"title": "Read-only API container status inspection",
"owner_agent": "SRE Agent",
"status": "waiting_control_path_readback",
"required_output": "api_container_state_and_redacted_health_log_tail",
},
{
"id": "stockplatform-api-targeted-reconcile",
"title": "Targeted API container reconcile",
"owner_agent": "DevOps Agent",
"status": "waiting_read_only_inspection",
"required_output": "bounded_api_reconcile_receipt_no_daemon_restart",
},
{
"id": "stockplatform-api-post-apply-verifier",
"title": "Public API post-apply verifier",
"owner_agent": "Verifier Agent",
"status": "ready",
"required_output": "public_health_freshness_ingestion_green",
},
{
"id": "stockplatform-api-learning-writeback",
"title": "KM / RAG / MCP / PlayBook trust writeback",
"owner_agent": "Learning Agent",
"status": "ready",
"required_output": "metadata_only_recovery_receipt_and_trust_delta",
},
]
def _controlled_action_candidates(
*,
candidate_packaged: bool,
runtime_ready: bool,
api_layer_classification: str,
) -> list[dict[str, Any]]:
if runtime_ready:
return []
if api_layer_classification == "api_live_data_dependency_not_ready":
return [
{
"id": "read_only_data_dependency_inspection",
"execution_route": "stockplatform_host_controlled_runtime_executor",
"current_status": (
"queued_waiting_control_path_readback"
if candidate_packaged
else "blocked_waiting_route_evidence"
),
"check_mode": "read_only_freshness_ingestion_and_postgres_contract",
"apply_allowed_when": [
"host_control_path_liveness_readback_ok",
"stockplatform_api_health_ok",
"source_freshness_contract_diff_present",
"post_apply_public_verifier_ready",
],
"forbidden_actions": [
"secret_read",
"docker_daemon_restart",
"host_reboot",
"database_write",
"fake_freshness_marker",
],
},
{
"id": "source_freshness_contract_preflight",
"execution_route": "stockplatform_data_contract_executor",
"current_status": "waiting_read_only_data_dependency_inspection",
"check_mode": "dry_run_source_and_runtime_contract_diff",
"bounded_apply": (
"apply only metadata/config contract repair after dry-run; "
"no DB write, no fake freshness marker, no destructive restore"
),
},
]
return [
{
"id": "read_only_container_inspection",
"execution_route": "stockplatform_host_controlled_runtime_executor",
"current_status": (
"queued_waiting_control_path_readback"
if candidate_packaged
else "blocked_waiting_route_evidence"
),
"check_mode": "read_only_status_and_redacted_logs",
"apply_allowed_when": [
"host_control_path_liveness_readback_ok",
"compose_project_or_container_target_matches_stockplatform_api",
"secret_scrubber_enabled_for_log_tail",
],
"forbidden_actions": [
"secret_read",
"docker_daemon_restart",
"host_reboot",
"database_write",
],
},
{
"id": "targeted_api_container_reconcile",
"execution_route": "stockplatform_host_controlled_runtime_executor",
"current_status": "waiting_read_only_container_inspection",
"check_mode": "dry_run_target_selector_and_current_state_diff",
"apply_allowed_when": [
"api_container_missing_exited_or_unhealthy",
"current_image_and_compose_source_match_expected_runtime",
"post_apply_public_verifier_ready",
"rollback_receipt_path_ready",
],
"bounded_apply": (
"reconcile only the StockPlatform API container/service target; "
"no Docker daemon restart, no host reboot, no DB write"
),
},
]
def _post_apply_verifiers(*, runtime_ready: bool) -> list[dict[str, Any]]:
if runtime_ready:
return []
return [
{
"id": "stockplatform_public_healthz",
"url": "https://stock.wooo.work/api/healthz",
"expected_http_status": 200,
},
{
"id": "stockplatform_freshness",
"url": "https://stock.wooo.work/api/v1/system/freshness",
"expected_status": "ok",
},
{
"id": "stockplatform_ingestion",
"url": "https://stock.wooo.work/api/v1/system/ingestion",
"expected_status": "ok",
},
{
"id": "awoooi_priority_work_order_readback",
"url": "/api/v1/agents/awoooi-priority-work-order-readback",
"expected_rollup": "stockplatform_public_api_runtime_ready=true",
},
]
def _learning_writeback_contracts() -> list[dict[str, Any]]:
targets = [
"timeline_event",
"knowledge_entry",
"rag_embedding",
"mcp_tool_registry_signal",
"playbook_trust_score",
"controlled_recovery_report",
]
return [
{
"target": target,
"writeback_mode": "metadata_only",
"raw_log_allowed": False,
"secret_value_allowed": False,
"receipt_key": f"stockplatform_public_api_recovery::{target}",
}
for target in targets
]
def _dict(value: Any) -> dict[str, Any]:
return value if isinstance(value, dict) else {}
def _int(value: Any) -> int:
try:
return int(str(value))
except (TypeError, ValueError):
return 0
def _strings(value: Any) -> list[str]:
if not isinstance(value, list):
return []
return [str(item) for item in value if item is not None]
def _unique_strings(values: list[str]) -> list[str]:
seen: set[str] = set()
unique: list[str] = []
for value in values:
if value in seen:
continue
seen.add(value)
unique.append(value)
return unique

View File

@@ -10,9 +10,13 @@ from fastapi.testclient import TestClient
from src.api.v1 import agents
from src.api.v1.agents import router
from src.services.awoooi_priority_work_order_readback import (
apply_stockplatform_public_api_controlled_recovery_preflight,
apply_stockplatform_public_api_runtime_readback,
load_latest_awoooi_priority_work_order_readback,
)
from src.services.stockplatform_public_api_controlled_recovery_preflight import (
load_latest_stockplatform_public_api_controlled_recovery_preflight,
)
_REPO_ROOT = Path(__file__).resolve().parents[3]
_SNAPSHOT_PATH = (
@@ -135,6 +139,77 @@ def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift()
)
def test_awoooi_priority_work_order_readback_overlays_controlled_recovery_preflight():
payload = load_latest_awoooi_priority_work_order_readback()
runtime = _stockplatform_runtime_blocked()
preflight = load_latest_stockplatform_public_api_controlled_recovery_preflight(
runtime_readback=runtime
)
apply_stockplatform_public_api_runtime_readback(payload, runtime)
apply_stockplatform_public_api_controlled_recovery_preflight(payload, preflight)
state = payload["mainline_execution_state"]
in_progress = payload["in_progress_or_blocked_in_priority_order"][0]
evidence = in_progress["evidence"]
assert payload["status"] == (
"p0_006_blocked_stockplatform_public_api_controlled_recovery_preflight"
)
assert state["active_p0_state"] == (
"blocked_stockplatform_public_api_controlled_recovery_preflight"
)
assert state["next_executable_mainline_workplan_id"] == (
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT"
)
assert evidence["stockplatform_controlled_recovery_preflight_status"] == (
"controlled_recovery_preflight_packaged_waiting_control_path_readback"
)
assert evidence["stockplatform_controlled_recovery_candidate_packaged"] is True
assert evidence["stockplatform_host_control_path_readback_required"] is True
assert evidence["stockplatform_controlled_work_item_count"] == 5
assert evidence["stockplatform_post_apply_verifier_count"] == 4
assert evidence["stockplatform_learning_writeback_contract_count"] == 6
assert evidence["stockplatform_current_apply_allowed"] is False
assert evidence["stockplatform_runtime_write_gate"] == "controlled_after_preflight"
assert "not a manual end state" in in_progress["reason"]
assert "Do not restart Docker daemon" in in_progress["professional_fix"]["action"]
assert payload["summary"][
"stockplatform_public_api_controlled_recovery_candidate_packaged"
] is True
assert payload["next_execution_order"][0].startswith(
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT"
)
def test_awoooi_priority_work_order_readback_routes_stockplatform_data_dependency():
payload = load_latest_awoooi_priority_work_order_readback()
runtime = _stockplatform_runtime_data_dependency_blocked()
preflight = load_latest_stockplatform_public_api_controlled_recovery_preflight(
runtime_readback=runtime
)
apply_stockplatform_public_api_runtime_readback(payload, runtime)
apply_stockplatform_public_api_controlled_recovery_preflight(payload, preflight)
state = payload["mainline_execution_state"]
in_progress = payload["in_progress_or_blocked_in_priority_order"][0]
evidence = in_progress["evidence"]
assert evidence["stockplatform_controlled_recovery_preflight_status"] == (
"controlled_data_dependency_preflight_packaged_waiting_control_path_readback"
)
assert evidence["stockplatform_controlled_recovery_api_layer"] == (
"api_live_data_dependency_not_ready"
)
assert state["next_executable_mainline_state"] == (
"controlled_recovery_preflight_waiting_stockplatform_data_dependency_"
"readback_and_source_freshness_contract"
)
assert "data dependency/source freshness" in in_progress["reason"]
assert "Do not restart Docker daemon" in in_progress["professional_fix"]["action"]
assert "write DB rows" in in_progress["professional_fix"]["action"]
assert "do not DB-write or fake freshness" in payload["next_execution_order"][0]
def test_awoooi_priority_work_order_readback_normalizes_runtime_source_truth(
monkeypatch: pytest.MonkeyPatch,
):
@@ -230,6 +305,7 @@ def _stockplatform_runtime_blocked() -> dict:
"runtime_ready": False,
"live_drift_from_committed_scorecard": True,
"readback": {
"web_health_http_status": 200,
"api_health_http_status": 502,
"freshness_http_status": 502,
"ingestion_http_status": 502,
@@ -245,3 +321,35 @@ def _stockplatform_runtime_blocked() -> dict:
],
},
}
def _stockplatform_runtime_data_dependency_blocked() -> dict:
return {
"schema_version": "stockplatform_public_api_runtime_readback_v1",
"status": "blocked_stockplatform_public_api_runtime_drift",
"safe_next_step": "run_stockplatform_data_dependency_controlled_preflight",
"active_blockers": [
"stockplatform_freshness_postgres_not_ready",
"stockplatform_ingestion_postgres_not_ready",
"stockplatform_recovery_control_path_receipt_missing",
"stockplatform_controlled_deploy_or_ssh_readback_required",
],
"runtime_ready": False,
"live_drift_from_committed_scorecard": True,
"readback": {
"web_health_http_status": 200,
"api_health_http_status": 200,
"freshness_http_status": 200,
"ingestion_http_status": 200,
},
"rollups": {
"http_502_count": 0,
},
"recovery_control_path": {
"status": "blocked_recovery_control_path_receipt_missing",
"active_blockers": [
"stockplatform_recovery_control_path_receipt_missing",
"stockplatform_controlled_deploy_or_ssh_readback_required",
],
},
}

View File

@@ -0,0 +1,233 @@
from __future__ import annotations
from pathlib import Path
from fastapi import FastAPI
from fastapi.testclient import TestClient
from src.api.v1 import agents
from src.api.v1.agents import router
from src.services.stockplatform_public_api_controlled_recovery_preflight import (
load_latest_stockplatform_public_api_controlled_recovery_preflight,
)
def test_stockplatform_controlled_recovery_preflight_packages_api_502(
tmp_path: Path,
):
payload = load_latest_stockplatform_public_api_controlled_recovery_preflight(
runtime_readback=_runtime_blocked(),
route_source_path=_route_source(tmp_path),
)
assert (
payload["schema_version"]
== "stockplatform_public_api_controlled_recovery_preflight_v1"
)
assert payload["priority"] == "P0-006"
assert payload["status"] == (
"controlled_recovery_preflight_packaged_waiting_control_path_readback"
)
assert payload["api_layer_classification"] == (
"edge_alive_api_upstream_unavailable"
)
assert payload["target_selector"]["route_target_ready"] is True
assert payload["source_of_truth_diff"]["status"] == (
"route_source_present_runtime_api_upstream_blocked"
)
assert (
payload["controlled_apply_policy"]["runtime_write_gate"]
== "controlled_after_preflight"
)
assert payload["controlled_apply_policy"]["break_glass_required"] is False
assert payload["controlled_apply_policy"]["current_apply_allowed"] is False
assert "stockplatform_110_control_path_readback_required" in payload[
"active_blockers"
]
assert payload["rollups"]["controlled_recovery_candidate_packaged"] is True
assert payload["rollups"]["host_control_path_readback_required"] is True
assert payload["rollups"]["controlled_work_item_count"] == 5
assert payload["rollups"]["post_apply_verifier_count"] == 4
assert payload["rollups"]["learning_writeback_contract_count"] == 6
assert payload["controlled_work_items"][0]["status"] == (
"queued_waiting_control_path_readback"
)
assert payload["post_apply_verifiers"][0]["expected_http_status"] == 200
assert payload["learning_writeback_contracts"][0]["writeback_mode"] == (
"metadata_only"
)
assert payload["operation_boundaries"]["ssh_used"] is False
assert payload["operation_boundaries"]["docker_command_performed"] is False
assert payload["operation_boundaries"]["docker_daemon_restart_performed"] is False
assert payload["operation_boundaries"]["host_reboot_performed"] is False
assert payload["operation_boundaries"]["database_write_or_restore_performed"] is False
assert payload["operation_boundaries"]["secret_value_collection_allowed"] is False
def test_stockplatform_controlled_recovery_preflight_not_required_when_ready(
tmp_path: Path,
):
payload = load_latest_stockplatform_public_api_controlled_recovery_preflight(
runtime_readback=_runtime_ready(),
route_source_path=_route_source(tmp_path),
)
assert payload["status"] == "stockplatform_public_api_recovery_preflight_not_required"
assert payload["active_blockers"] == []
assert payload["controlled_work_items"] == []
assert payload["rollups"]["recovery_required"] is False
def test_stockplatform_controlled_recovery_preflight_packages_data_dependency(
tmp_path: Path,
):
payload = load_latest_stockplatform_public_api_controlled_recovery_preflight(
runtime_readback=_runtime_data_dependency_blocked(),
route_source_path=_route_source(tmp_path),
)
assert payload["status"] == (
"controlled_data_dependency_preflight_packaged_waiting_control_path_readback"
)
assert payload["api_layer_classification"] == "api_live_data_dependency_not_ready"
assert payload["source_of_truth_diff"]["status"] == (
"route_source_present_api_live_data_dependency_not_ready"
)
assert payload["controlled_apply_policy"]["current_apply_blocker"] == (
"data_dependency_control_path_readback_required_before_source_"
"freshness_or_postgres_contract_apply"
)
assert "read_only_data_dependency_status_inspection" in payload[
"controlled_apply_policy"
]["allowed_scope"]
assert "fake_freshness_marker" in payload["controlled_action_candidates"][0][
"forbidden_actions"
]
assert payload["controlled_work_items"][1]["id"] == (
"stockplatform-data-dependency-inspection"
)
assert payload["controlled_action_candidates"][0]["id"] == (
"read_only_data_dependency_inspection"
)
assert payload["rollups"]["controlled_recovery_candidate_packaged"] is True
assert payload["operation_boundaries"]["database_write_or_restore_performed"] is False
def test_stockplatform_controlled_recovery_preflight_endpoint_returns_payload(
monkeypatch,
tmp_path: Path,
):
monkeypatch.setattr(
agents,
"load_latest_stockplatform_public_api_controlled_recovery_preflight",
lambda: load_latest_stockplatform_public_api_controlled_recovery_preflight(
runtime_readback=_runtime_blocked(),
route_source_path=_route_source(tmp_path),
),
)
app = FastAPI()
app.include_router(router, prefix="/api/v1")
client = TestClient(app)
response = client.get(
"/api/v1/agents/stockplatform-public-api-controlled-recovery-preflight"
)
assert response.status_code == 200
data = response.json()
assert data["status"] == (
"controlled_recovery_preflight_packaged_waiting_control_path_readback"
)
assert data["rollups"]["controlled_recovery_candidate_packaged"] is True
def _route_source(tmp_path: Path) -> Path:
path = tmp_path / "188-all-sites.conf.j2"
path.write_text(
"""
# Stock Platform - stock.wooo.work
server {
server_name stock.wooo.work;
location / {
proxy_pass http://192.168.0.110:31235;
}
}
# Next Site
""",
encoding="utf-8",
)
return path
def _runtime_blocked() -> dict:
return {
"schema_version": "stockplatform_public_api_runtime_readback_v1",
"priority": "P0-006",
"status": "blocked_stockplatform_public_api_runtime_drift",
"safe_next_step": (
"recover_stockplatform_api_container_runtime_or_docker_control_path_"
"without_daemon_restart_then_rerun_public_api_readback"
),
"active_blockers": [
"stockplatform_public_api_healthz_http_502",
"stockplatform_freshness_http_502",
"stockplatform_ingestion_http_502",
],
"runtime_ready": False,
"base_url": "https://stock.wooo.work",
"readback": {
"web_health_http_status": 200,
"api_health_http_status": 502,
"freshness_http_status": 502,
"ingestion_http_status": 502,
},
"rollups": {
"http_502_count": 3,
},
}
def _runtime_ready() -> dict:
return {
"schema_version": "stockplatform_public_api_runtime_readback_v1",
"priority": "P0-006",
"status": "stockplatform_public_api_runtime_ready",
"runtime_ready": True,
"base_url": "https://stock.wooo.work",
"active_blockers": [],
"readback": {
"web_health_http_status": 200,
"api_health_http_status": 200,
"freshness_http_status": 200,
"ingestion_http_status": 200,
},
"rollups": {
"http_502_count": 0,
},
}
def _runtime_data_dependency_blocked() -> dict:
return {
"schema_version": "stockplatform_public_api_runtime_readback_v1",
"priority": "P0-006",
"status": "blocked_stockplatform_public_api_runtime_drift",
"safe_next_step": "run_stockplatform_data_dependency_controlled_preflight",
"active_blockers": [
"stockplatform_freshness_postgres_not_ready",
"stockplatform_ingestion_postgres_not_ready",
"stockplatform_recovery_control_path_receipt_missing",
"stockplatform_controlled_deploy_or_ssh_readback_required",
],
"runtime_ready": False,
"base_url": "https://stock.wooo.work",
"readback": {
"web_health_http_status": 200,
"api_health_http_status": 200,
"freshness_http_status": 200,
"ingestion_http_status": 200,
},
"rollups": {
"http_502_count": 0,
},
}

View File

@@ -50299,3 +50299,19 @@ production browser smoke:
- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth。
- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。
- 沒有重啟主機,沒有 workflow_dispatch沒有 host / Docker / K8s / DB / firewall runtime 寫入。
## 2026-06-30 — 16:45 P0-006 StockPlatform API controlled recovery preflight
**完成內容**
- 新增 `stockplatform_public_api_controlled_recovery_preflight`,把 StockPlatform public API 502 從單純 readback 推進為 AI controlled recovery preflight輸出 target selector、source-of-truth diff、API layer classification、controlled action candidates、post-apply verifiers 與 KM / RAG / MCP / PlayBook metadata writeback contracts。
- 新增 production route `GET /api/v1/agents/stockplatform-public-api-controlled-recovery-preflight`;端點只讀 public HTTPS readback 與 repo 內 188 Nginx source-of-truth不 SSH、不 Docker、不 restart、不寫 DB、不讀 secret、不觸發 workflow。
- `awoooi_priority_work_order_readback` 新增 recovery preflight overlay當 StockPlatform public web 200 但 public API/freshness/ingestion 502 時,下一個主線工作會前進到 `P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT`,不是停在人工終局。
- Gitea CD controlled-runtime 白名單補入新 preflight service / test避免 P0-006 窄 API 變更誤跑 full/B5 profile。
**受控邊界**
- 允許候選:恢復/驗證 110 control path readback、read-only API container inspection、redacted log tail、必要時僅針對 StockPlatform API container 做 bounded reconcile並以 public API health/freshness/ingestion + priority readback 驗證。
- 禁止事項維持Docker daemon restart、host reboot、Nginx reload/restart、DB write/restore/prune、secret value read、workflow dispatch、force push / repo refs mutation。
**仍需驗證**
- 跑 focused API tests、py_compile、ruff、Gitea runner pressure guard、Gitea secret step guard 與 `git diff --check`
- 推送後等待 deploy marker再讀回 priority endpoint 與新 preflight endpoint確認 production 顯示 recovery preflight 而非 manual/default-human 終局。

View File

@@ -320,13 +320,17 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() -
"apps/api/src/services/reboot_auto_recovery_slo_scorecard.py)",
"apps/api/src/services/reboot_auto_recovery_drill_preflight.py)",
"apps/api/src/services/stockplatform_public_api_runtime_readback.py)",
"apps/api/src/services/stockplatform_public_api_controlled_recovery_preflight.py)",
"apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py)",
"apps/api/tests/test_stockplatform_public_api_runtime_readback.py)",
"apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py)",
"src/services/reboot_auto_recovery_slo_scorecard.py",
"src/services/reboot_auto_recovery_drill_preflight.py",
"src/services/stockplatform_public_api_runtime_readback.py",
"src/services/stockplatform_public_api_controlled_recovery_preflight.py",
"tests/test_reboot_auto_recovery_slo_scorecard_api.py",
"tests/test_stockplatform_public_api_runtime_readback.py",
"tests/test_stockplatform_public_api_controlled_recovery_preflight.py",
"scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service)",
"scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer)",
"scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh)",