feat(api): expose stockplatform controlled recovery preflight
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
This commit is contained in:
@@ -336,6 +336,8 @@ jobs:
|
||||
;;
|
||||
apps/api/src/services/stockplatform_public_api_runtime_readback.py)
|
||||
;;
|
||||
apps/api/src/services/stockplatform_public_api_controlled_recovery_preflight.py)
|
||||
;;
|
||||
apps/api/src/services/iwooos_security_operating_system.py)
|
||||
;;
|
||||
apps/api/Dockerfile)
|
||||
@@ -440,6 +442,8 @@ jobs:
|
||||
;;
|
||||
apps/api/tests/test_stockplatform_public_api_runtime_readback.py)
|
||||
;;
|
||||
apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py)
|
||||
;;
|
||||
apps/api/tests/test_iwooos_security_operating_system.py)
|
||||
;;
|
||||
apps/api/tests/test_iwooos_wazuh_prod_manifest.py)
|
||||
@@ -635,6 +639,7 @@ jobs:
|
||||
src/services/reboot_auto_recovery_slo_scorecard.py \
|
||||
src/services/reboot_auto_recovery_drill_preflight.py \
|
||||
src/services/stockplatform_public_api_runtime_readback.py \
|
||||
src/services/stockplatform_public_api_controlled_recovery_preflight.py \
|
||||
src/services/iwooos_security_operating_system.py \
|
||||
src/services/awoooi_gitea_onboarding_warning_step_dashboard.py \
|
||||
src/services/awoooi_gitea_onboarding_warning_step_owner_package.py \
|
||||
@@ -693,6 +698,7 @@ jobs:
|
||||
tests/test_gitea_workflow_runner_owner_attestation_request_api.py \
|
||||
tests/test_reboot_auto_recovery_slo_scorecard_api.py \
|
||||
tests/test_stockplatform_public_api_runtime_readback.py \
|
||||
tests/test_stockplatform_public_api_controlled_recovery_preflight.py \
|
||||
tests/test_iwooos_security_operating_system.py \
|
||||
tests/test_awoooi_production_deploy_readback_blocker.py \
|
||||
tests/test_awoooi_priority_work_order_readback_api.py \
|
||||
|
||||
@@ -335,6 +335,7 @@ from src.services.awoooi_gitea_onboarding_warning_step_template_copy_receipt imp
|
||||
load_latest_awoooi_gitea_onboarding_warning_step_template_copy_receipt,
|
||||
)
|
||||
from src.services.awoooi_priority_work_order_readback import (
|
||||
apply_stockplatform_public_api_controlled_recovery_preflight,
|
||||
apply_stockplatform_public_api_runtime_readback,
|
||||
load_latest_awoooi_priority_work_order_readback,
|
||||
)
|
||||
@@ -443,6 +444,9 @@ from src.services.service_health_failure_notification_policy import (
|
||||
from src.services.service_health_gap_matrix import (
|
||||
load_latest_service_health_gap_matrix,
|
||||
)
|
||||
from src.services.stockplatform_public_api_controlled_recovery_preflight import (
|
||||
load_latest_stockplatform_public_api_controlled_recovery_preflight,
|
||||
)
|
||||
from src.services.stockplatform_public_api_runtime_readback import (
|
||||
load_latest_stockplatform_public_api_runtime_readback,
|
||||
)
|
||||
@@ -1086,6 +1090,14 @@ async def get_awoooi_priority_work_order_readback() -> dict[str, Any]:
|
||||
load_latest_stockplatform_public_api_runtime_readback
|
||||
)
|
||||
apply_stockplatform_public_api_runtime_readback(payload, stockplatform_runtime)
|
||||
stockplatform_recovery = await asyncio.to_thread(
|
||||
load_latest_stockplatform_public_api_controlled_recovery_preflight,
|
||||
runtime_readback=stockplatform_runtime,
|
||||
)
|
||||
apply_stockplatform_public_api_controlled_recovery_preflight(
|
||||
payload,
|
||||
stockplatform_recovery,
|
||||
)
|
||||
return redact_public_lan_topology(payload)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
@@ -1128,6 +1140,36 @@ async def get_stockplatform_public_api_runtime_readback() -> dict[str, Any]:
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/stockplatform-public-api-controlled-recovery-preflight",
|
||||
response_model=dict[str, Any],
|
||||
summary="取得 StockPlatform public API controlled recovery preflight",
|
||||
description=(
|
||||
"讀取 StockPlatform public API runtime drift,並產生 AI controlled "
|
||||
"recovery preflight:target selector、source-of-truth diff、bounded "
|
||||
"action candidates、rollback/verifier 與 KM/RAG/MCP/PlayBook writeback "
|
||||
"contract。此端點不 SSH、不 Docker、不 restart、不寫 DB、不讀 secret、"
|
||||
"不觸發 workflow。"
|
||||
),
|
||||
)
|
||||
async def get_stockplatform_public_api_controlled_recovery_preflight() -> dict[str, Any]:
|
||||
"""回傳 StockPlatform public API controlled recovery preflight。"""
|
||||
try:
|
||||
payload = await asyncio.to_thread(
|
||||
load_latest_stockplatform_public_api_controlled_recovery_preflight
|
||||
)
|
||||
return redact_public_lan_topology(payload)
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
logger.error(
|
||||
"stockplatform_public_api_controlled_recovery_preflight_invalid",
|
||||
error=str(exc),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="StockPlatform public API controlled recovery preflight 無效",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/product-awoooi-manifest-standard",
|
||||
response_model=dict[str, Any],
|
||||
|
||||
@@ -184,6 +184,247 @@ def apply_stockplatform_public_api_runtime_readback(
|
||||
_refresh_rollups_after_stockplatform_overlay(payload, state)
|
||||
|
||||
|
||||
def apply_stockplatform_public_api_controlled_recovery_preflight(
|
||||
payload: dict[str, Any],
|
||||
recovery_preflight: dict[str, Any],
|
||||
) -> None:
|
||||
"""Overlay the packaged StockPlatform controlled recovery preflight."""
|
||||
status = str(recovery_preflight.get("status") or "unknown")
|
||||
if status == "stockplatform_public_api_recovery_preflight_not_required":
|
||||
return
|
||||
|
||||
state = _dict(payload.setdefault("mainline_execution_state", {}))
|
||||
rollups = _dict(recovery_preflight.get("rollups"))
|
||||
source_diff = _dict(recovery_preflight.get("source_of_truth_diff"))
|
||||
target_selector = _dict(recovery_preflight.get("target_selector"))
|
||||
controlled_policy = _dict(recovery_preflight.get("controlled_apply_policy"))
|
||||
candidate_packaged = bool(
|
||||
rollups.get("controlled_recovery_candidate_packaged") is True
|
||||
)
|
||||
api_layer = str(
|
||||
recovery_preflight.get("api_layer_classification") or "unknown"
|
||||
)
|
||||
|
||||
state["stockplatform_public_api_controlled_recovery_preflight_status"] = status
|
||||
state["stockplatform_public_api_controlled_recovery_api_layer"] = api_layer
|
||||
state["stockplatform_public_api_controlled_recovery_active_blockers"] = _strings(
|
||||
recovery_preflight.get("active_blockers")
|
||||
)
|
||||
state["active_p0_state"] = (
|
||||
"blocked_stockplatform_public_api_controlled_recovery_preflight"
|
||||
)
|
||||
state["next_executable_mainline_workplan_id"] = (
|
||||
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT"
|
||||
)
|
||||
state["next_executable_mainline_state"] = _stockplatform_recovery_next_state(
|
||||
candidate_packaged=candidate_packaged,
|
||||
api_layer=api_layer,
|
||||
)
|
||||
|
||||
existing_blockers = _strings(state.get("active_p0_live_active_blockers"))
|
||||
state["active_p0_live_active_blockers"] = _unique_strings(
|
||||
existing_blockers
|
||||
+ ["stockplatform_public_api_controlled_recovery_preflight"]
|
||||
+ _strings(recovery_preflight.get("active_blockers"))
|
||||
)
|
||||
|
||||
for item in _list(payload.get("in_progress_or_blocked_in_priority_order")):
|
||||
workplan = _dict(item)
|
||||
if workplan.get("workplan_id") != "P0-006":
|
||||
continue
|
||||
|
||||
evidence = _dict(workplan.setdefault("evidence", {}))
|
||||
evidence["stockplatform_controlled_recovery_preflight_status"] = status
|
||||
evidence["stockplatform_controlled_recovery_api_layer"] = state[
|
||||
"stockplatform_public_api_controlled_recovery_api_layer"
|
||||
]
|
||||
evidence["stockplatform_route_target_ready"] = bool(
|
||||
target_selector.get("route_target_ready") is True
|
||||
)
|
||||
evidence["stockplatform_source_of_truth_diff_status"] = str(
|
||||
source_diff.get("status") or ""
|
||||
)
|
||||
evidence["stockplatform_controlled_recovery_candidate_packaged"] = (
|
||||
candidate_packaged
|
||||
)
|
||||
evidence["stockplatform_host_control_path_readback_required"] = bool(
|
||||
rollups.get("host_control_path_readback_required") is True
|
||||
)
|
||||
evidence["stockplatform_controlled_work_item_count"] = _int(
|
||||
rollups.get("controlled_work_item_count")
|
||||
)
|
||||
evidence["stockplatform_post_apply_verifier_count"] = _int(
|
||||
rollups.get("post_apply_verifier_count")
|
||||
)
|
||||
evidence["stockplatform_learning_writeback_contract_count"] = _int(
|
||||
rollups.get("learning_writeback_contract_count")
|
||||
)
|
||||
evidence["stockplatform_current_apply_allowed"] = bool(
|
||||
controlled_policy.get("current_apply_allowed") is True
|
||||
)
|
||||
evidence["stockplatform_runtime_write_gate"] = str(
|
||||
controlled_policy.get("runtime_write_gate") or ""
|
||||
)
|
||||
|
||||
workplan["status"] = (
|
||||
"blocked_stockplatform_public_api_controlled_recovery_preflight"
|
||||
)
|
||||
workplan["safe_next_step"] = str(
|
||||
recovery_preflight.get("safe_next_step") or ""
|
||||
)
|
||||
workplan["reason"] = (
|
||||
_stockplatform_recovery_reason(
|
||||
candidate_packaged=candidate_packaged,
|
||||
api_layer=api_layer,
|
||||
)
|
||||
)
|
||||
professional_fix = _dict(workplan.setdefault("professional_fix", {}))
|
||||
professional_fix["action"] = _stockplatform_recovery_action(
|
||||
api_layer=api_layer,
|
||||
)
|
||||
professional_fix["owner"] = (
|
||||
"StockPlatform API controlled recovery preflight plus P0-006 "
|
||||
"priority readback"
|
||||
)
|
||||
|
||||
payload["status"] = (
|
||||
"p0_006_blocked_stockplatform_public_api_controlled_recovery_preflight"
|
||||
)
|
||||
payload["next_execution_order"] = [
|
||||
_stockplatform_recovery_first_order(api_layer=api_layer),
|
||||
(
|
||||
"P0-006-STOCKPLATFORM-PUBLIC-API-RUNTIME-READBACK: rerun public "
|
||||
"health/freshness/ingestion after the bounded reconcile candidate; "
|
||||
"public API must be green before returning to reboot-window proof."
|
||||
),
|
||||
(
|
||||
"P0-006: keep reboot SLO timer live, but do not claim final SLO "
|
||||
"closure while StockPlatform public API readback is blocked."
|
||||
),
|
||||
(
|
||||
"NEXT: after StockPlatform public API readback is green or a "
|
||||
"separate hard-blocker handoff is recorded, resume the next "
|
||||
"blocker-free priority item."
|
||||
),
|
||||
]
|
||||
_refresh_rollups_after_stockplatform_overlay(payload, state)
|
||||
summary = _dict(payload.setdefault("summary", {}))
|
||||
summary["stockplatform_public_api_controlled_recovery_preflight_status"] = status
|
||||
summary["stockplatform_public_api_controlled_recovery_api_layer"] = str(
|
||||
recovery_preflight.get("api_layer_classification") or "unknown"
|
||||
)
|
||||
summary["stockplatform_public_api_controlled_recovery_candidate_packaged"] = bool(
|
||||
candidate_packaged
|
||||
)
|
||||
summary["stockplatform_public_api_host_control_path_readback_required"] = bool(
|
||||
rollups.get("host_control_path_readback_required") is True
|
||||
)
|
||||
|
||||
|
||||
def _stockplatform_recovery_next_state(
|
||||
*,
|
||||
candidate_packaged: bool,
|
||||
api_layer: str,
|
||||
) -> str:
|
||||
if api_layer == "api_live_data_dependency_not_ready":
|
||||
return (
|
||||
"controlled_recovery_preflight_waiting_stockplatform_data_dependency_"
|
||||
"readback_and_source_freshness_contract"
|
||||
)
|
||||
if candidate_packaged:
|
||||
return (
|
||||
"controlled_recovery_packaged_waiting_stockplatform_110_control_path_"
|
||||
"readback_and_read_only_container_inspection"
|
||||
)
|
||||
if api_layer == "public_edge_or_route_unavailable":
|
||||
return (
|
||||
"controlled_recovery_preflight_waiting_public_edge_or_188_to_110_"
|
||||
"upstream_reprobe_before_targeted_apply"
|
||||
)
|
||||
return "controlled_recovery_preflight_waiting_route_or_runtime_evidence"
|
||||
|
||||
|
||||
def _stockplatform_recovery_reason(
|
||||
*,
|
||||
candidate_packaged: bool,
|
||||
api_layer: str,
|
||||
) -> str:
|
||||
if api_layer == "api_live_data_dependency_not_ready":
|
||||
return (
|
||||
"Live StockPlatform API health is reachable, but freshness and/or "
|
||||
"ingestion readbacks are not ok. The P0-006 lane now has a "
|
||||
"controlled data-dependency preflight with target selector, "
|
||||
"source-of-truth diff, data dependency work items, post-apply "
|
||||
"verifiers, and KM/RAG/MCP/PlayBook writeback contracts. The "
|
||||
"remaining blocker is data dependency/source freshness control-path "
|
||||
"readback, not a manual end state."
|
||||
)
|
||||
if candidate_packaged:
|
||||
return (
|
||||
"Live StockPlatform public web is reachable but public API routes "
|
||||
"return 502. The P0-006 lane now has a controlled recovery package "
|
||||
"with target selector, source-of-truth diff, bounded action "
|
||||
"candidates, post-apply verifiers, and KM/RAG/MCP/PlayBook "
|
||||
"writeback contracts. The remaining blocker is the StockPlatform "
|
||||
"110 control-path readback, not a manual end state."
|
||||
)
|
||||
if api_layer == "public_edge_or_route_unavailable":
|
||||
return (
|
||||
"Live StockPlatform public API recovery preflight is active, but "
|
||||
"the current public probe cannot prove the edge-alive/API-upstream "
|
||||
"502 shape. The next AI-controlled step is a bounded public-edge "
|
||||
"and 188-to-110 upstream reprobe before any targeted apply; this is "
|
||||
"not a manual end state."
|
||||
)
|
||||
return (
|
||||
"StockPlatform public API recovery preflight is active and waiting for "
|
||||
"route/runtime evidence before any bounded apply; this is not a manual "
|
||||
"end state."
|
||||
)
|
||||
|
||||
|
||||
def _stockplatform_recovery_action(*, api_layer: str) -> str:
|
||||
if api_layer == "api_live_data_dependency_not_ready":
|
||||
return (
|
||||
"Run the StockPlatform data dependency controlled preflight in order: "
|
||||
"restore/control the 110 host readback path, inspect freshness, "
|
||||
"ingestion, source freshness and Postgres contract status in read-only "
|
||||
"mode, apply only metadata/config contract repair after dry-run if "
|
||||
"needed, then rerun public health/freshness/ingestion and write "
|
||||
"metadata-only KM/RAG/MCP/PlayBook receipts. Do not restart Docker "
|
||||
"daemon, reboot hosts, reload Nginx, write DB rows, fake freshness, "
|
||||
"trigger workflows, or read secrets from this lane."
|
||||
)
|
||||
return (
|
||||
"Run the StockPlatform API controlled recovery preflight in order: "
|
||||
"restore/control the 110 host readback path, perform read-only API "
|
||||
"container inspection with redacted logs, reconcile only the API "
|
||||
"container if inspection proves it is missing/exited/unhealthy, then "
|
||||
"rerun public health/freshness/ingestion and write metadata-only "
|
||||
"KM/RAG/MCP/PlayBook receipts. Do not restart Docker daemon, reboot "
|
||||
"hosts, reload Nginx, write DB rows, trigger workflows, or read "
|
||||
"secrets from this lane."
|
||||
)
|
||||
|
||||
|
||||
def _stockplatform_recovery_first_order(*, api_layer: str) -> str:
|
||||
if api_layer == "api_live_data_dependency_not_ready":
|
||||
return (
|
||||
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT: "
|
||||
"StockPlatform API health is reachable, so the active mainline item "
|
||||
"is data dependency/source freshness control-path readback, "
|
||||
"read-only contract inspection, verifier, and metadata writeback; "
|
||||
"do not DB-write or fake freshness from this lane."
|
||||
)
|
||||
return (
|
||||
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT: "
|
||||
"controlled recovery package is now the active mainline item; "
|
||||
"restore/control the 110 readback path, inspect the API container, "
|
||||
"and only then allow a targeted API reconcile with verifier and "
|
||||
"metadata writeback."
|
||||
)
|
||||
|
||||
|
||||
def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
from src.services.awoooi_gitea_onboarding_warning_step_runtime_enablement_gate import (
|
||||
load_latest_awoooi_gitea_onboarding_warning_step_runtime_enablement_gate,
|
||||
|
||||
@@ -0,0 +1,614 @@
|
||||
"""StockPlatform public API controlled recovery preflight.
|
||||
|
||||
This module packages the next P0-006 action after public API 502 drift is
|
||||
detected. It does not SSH, run Docker, restart services, write databases, read
|
||||
secrets, or trigger workflows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from src.services.stockplatform_public_api_runtime_readback import (
|
||||
load_latest_stockplatform_public_api_runtime_readback,
|
||||
)
|
||||
|
||||
_SCHEMA_VERSION = "stockplatform_public_api_controlled_recovery_preflight_v1"
|
||||
_DEFAULT_ROUTE_SOURCE = (
|
||||
Path(__file__).resolve().parents[4]
|
||||
/ "infra"
|
||||
/ "ansible"
|
||||
/ "roles"
|
||||
/ "nginx"
|
||||
/ "templates"
|
||||
/ "188-all-sites.conf.j2"
|
||||
)
|
||||
|
||||
|
||||
def load_latest_stockplatform_public_api_controlled_recovery_preflight(
|
||||
*,
|
||||
runtime_readback: dict[str, Any] | None = None,
|
||||
route_source_path: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the controlled recovery preflight for the active P0-006 drift."""
|
||||
runtime = runtime_readback or load_latest_stockplatform_public_api_runtime_readback()
|
||||
route_contract = _load_route_contract(route_source_path or _DEFAULT_ROUTE_SOURCE)
|
||||
return _build_payload(runtime=runtime, route_contract=route_contract)
|
||||
|
||||
|
||||
def _build_payload(
|
||||
*,
|
||||
runtime: dict[str, Any],
|
||||
route_contract: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
runtime_ready = runtime.get("status") == "stockplatform_public_api_runtime_ready"
|
||||
readback = _dict(runtime.get("readback"))
|
||||
rollups = _dict(runtime.get("rollups"))
|
||||
api_layer_classification = _classify_api_layer(readback, rollups, runtime_ready)
|
||||
route_target_ready = route_contract.get("route_target_ready") is True
|
||||
recovery_required = not runtime_ready
|
||||
candidate_packaged = (
|
||||
recovery_required
|
||||
and api_layer_classification
|
||||
in {
|
||||
"edge_alive_api_upstream_unavailable",
|
||||
"api_live_data_dependency_not_ready",
|
||||
}
|
||||
and route_target_ready
|
||||
)
|
||||
active_blockers = _active_blockers(
|
||||
runtime=runtime,
|
||||
route_contract=route_contract,
|
||||
candidate_packaged=candidate_packaged,
|
||||
runtime_ready=runtime_ready,
|
||||
)
|
||||
status = _status(
|
||||
runtime_ready=runtime_ready,
|
||||
candidate_packaged=candidate_packaged,
|
||||
route_target_ready=route_target_ready,
|
||||
api_layer_classification=api_layer_classification,
|
||||
)
|
||||
|
||||
return {
|
||||
"schema_version": _SCHEMA_VERSION,
|
||||
"priority": "P0-006",
|
||||
"scope": "stockplatform_public_api_controlled_recovery_preflight",
|
||||
"status": status,
|
||||
"safe_next_step": _safe_next_step(
|
||||
runtime_ready=runtime_ready,
|
||||
candidate_packaged=candidate_packaged,
|
||||
route_target_ready=route_target_ready,
|
||||
api_layer_classification=api_layer_classification,
|
||||
),
|
||||
"active_blockers": active_blockers,
|
||||
"active_blocker_count": len(active_blockers),
|
||||
"api_layer_classification": api_layer_classification,
|
||||
"runtime_readback_status": str(runtime.get("status") or "unknown"),
|
||||
"target_selector": {
|
||||
"product_id": "stockplatform-v2",
|
||||
"public_domain": "stock.wooo.work",
|
||||
"public_base_url": str(runtime.get("base_url") or ""),
|
||||
"edge_route_owner": "188 nginx public ingress source-of-truth",
|
||||
"runtime_host_role": "stockplatform container runtime host",
|
||||
"route_source_file": str(route_contract.get("route_source_file") or ""),
|
||||
"route_target": str(route_contract.get("route_target") or ""),
|
||||
"route_target_ready": route_target_ready,
|
||||
"required_public_endpoints": [
|
||||
"/healthz",
|
||||
"/api/healthz",
|
||||
"/api/v1/system/freshness",
|
||||
"/api/v1/system/ingestion",
|
||||
],
|
||||
},
|
||||
"source_of_truth_diff": {
|
||||
"status": _source_of_truth_diff_status(
|
||||
runtime_ready=runtime_ready,
|
||||
route_target_ready=route_target_ready,
|
||||
api_layer_classification=api_layer_classification,
|
||||
),
|
||||
"public_web_health_http_status": readback.get("web_health_http_status"),
|
||||
"public_api_health_http_status": readback.get("api_health_http_status"),
|
||||
"freshness_http_status": readback.get("freshness_http_status"),
|
||||
"ingestion_http_status": readback.get("ingestion_http_status"),
|
||||
"http_502_count": _int(rollups.get("http_502_count")),
|
||||
"route_source_file": str(route_contract.get("route_source_file") or ""),
|
||||
"route_target": str(route_contract.get("route_target") or ""),
|
||||
},
|
||||
"controlled_apply_policy": {
|
||||
"risk_level": "high",
|
||||
"break_glass_required": False,
|
||||
"runtime_write_gate": "controlled_after_preflight",
|
||||
"controlled_recovery_candidate_packaged": candidate_packaged,
|
||||
"current_apply_allowed": False,
|
||||
"current_apply_blocker": _current_apply_blocker(
|
||||
runtime_ready=runtime_ready,
|
||||
candidate_packaged=candidate_packaged,
|
||||
api_layer_classification=api_layer_classification,
|
||||
),
|
||||
"allowed_scope": _allowed_scope(api_layer_classification),
|
||||
"forbidden_scope": [
|
||||
"docker_daemon_restart",
|
||||
"host_reboot",
|
||||
"nginx_reload_or_restart",
|
||||
"database_write_restore_or_prune",
|
||||
"secret_value_read",
|
||||
"workflow_dispatch",
|
||||
"force_push_or_repo_ref_mutation",
|
||||
],
|
||||
},
|
||||
"controlled_work_items": _controlled_work_items(
|
||||
candidate_packaged=candidate_packaged,
|
||||
runtime_ready=runtime_ready,
|
||||
api_layer_classification=api_layer_classification,
|
||||
),
|
||||
"controlled_action_candidates": _controlled_action_candidates(
|
||||
candidate_packaged=candidate_packaged,
|
||||
runtime_ready=runtime_ready,
|
||||
api_layer_classification=api_layer_classification,
|
||||
),
|
||||
"post_apply_verifiers": _post_apply_verifiers(runtime_ready=runtime_ready),
|
||||
"learning_writeback_contracts": _learning_writeback_contracts(),
|
||||
"rollups": {
|
||||
"runtime_ready": runtime_ready,
|
||||
"recovery_required": recovery_required,
|
||||
"route_target_ready": route_target_ready,
|
||||
"controlled_recovery_candidate_packaged": candidate_packaged,
|
||||
"controlled_recovery_preflight_ready": (
|
||||
runtime_ready or (candidate_packaged and not active_blockers)
|
||||
),
|
||||
"host_control_path_readback_required": (
|
||||
candidate_packaged and not runtime_ready
|
||||
),
|
||||
"controlled_work_item_count": len(
|
||||
_controlled_work_items(
|
||||
candidate_packaged=candidate_packaged,
|
||||
runtime_ready=runtime_ready,
|
||||
api_layer_classification=api_layer_classification,
|
||||
)
|
||||
),
|
||||
"post_apply_verifier_count": len(
|
||||
_post_apply_verifiers(runtime_ready=runtime_ready)
|
||||
),
|
||||
"learning_writeback_contract_count": len(_learning_writeback_contracts()),
|
||||
},
|
||||
"operation_boundaries": {
|
||||
"read_only_public_https_probe": True,
|
||||
"source_route_file_read": True,
|
||||
"ssh_used": False,
|
||||
"docker_command_performed": False,
|
||||
"docker_restart_performed": False,
|
||||
"docker_daemon_restart_performed": False,
|
||||
"host_reboot_performed": False,
|
||||
"nginx_reload_or_restart_performed": False,
|
||||
"database_write_or_restore_performed": False,
|
||||
"stockplatform_manual_data_write_performed": False,
|
||||
"secret_value_collection_allowed": False,
|
||||
"github_api_used": False,
|
||||
"workflow_trigger_performed": False,
|
||||
"runtime_write_allowed_by_this_readback": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _load_route_contract(path: Path) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"route_source_file": str(path),
|
||||
"route_target": "",
|
||||
"route_target_ready": False,
|
||||
"route_source_present": path.exists(),
|
||||
}
|
||||
if not path.exists():
|
||||
return payload
|
||||
|
||||
text = path.read_text(encoding="utf-8")
|
||||
stock_section = _stock_section(text)
|
||||
targets = re.findall(r"proxy_pass\s+(http://[^;]+);", stock_section)
|
||||
unique_targets = _unique_strings(targets)
|
||||
payload["route_target"] = unique_targets[0] if unique_targets else ""
|
||||
payload["route_targets"] = unique_targets
|
||||
payload["route_target_ready"] = bool(
|
||||
"server_name stock.wooo.work;" in stock_section and unique_targets
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _stock_section(text: str) -> str:
|
||||
start = text.find("# Stock Platform")
|
||||
if start == -1:
|
||||
return text
|
||||
next_marker = text.find("\n# =============================================================================\n# ", start + 80)
|
||||
if next_marker == -1:
|
||||
return text[start:]
|
||||
return text[start:next_marker]
|
||||
|
||||
|
||||
def _classify_api_layer(
|
||||
readback: dict[str, Any],
|
||||
rollups: dict[str, Any],
|
||||
runtime_ready: bool,
|
||||
) -> str:
|
||||
if runtime_ready:
|
||||
return "public_api_ready"
|
||||
if (
|
||||
readback.get("web_health_http_status") == 200
|
||||
and readback.get("api_health_http_status") == 502
|
||||
and _int(rollups.get("http_502_count")) >= 1
|
||||
):
|
||||
return "edge_alive_api_upstream_unavailable"
|
||||
if readback.get("api_health_http_status") == 200 and (
|
||||
readback.get("freshness_http_status") == 200
|
||||
or readback.get("ingestion_http_status") == 200
|
||||
):
|
||||
return "api_live_data_dependency_not_ready"
|
||||
return "public_edge_or_route_unavailable"
|
||||
|
||||
|
||||
def _source_of_truth_diff_status(
|
||||
*,
|
||||
runtime_ready: bool,
|
||||
route_target_ready: bool,
|
||||
api_layer_classification: str,
|
||||
) -> str:
|
||||
if runtime_ready:
|
||||
return "no_runtime_recovery_diff_required"
|
||||
if not route_target_ready:
|
||||
return "route_source_target_missing"
|
||||
if api_layer_classification == "edge_alive_api_upstream_unavailable":
|
||||
return "route_source_present_runtime_api_upstream_blocked"
|
||||
if api_layer_classification == "api_live_data_dependency_not_ready":
|
||||
return "route_source_present_api_live_data_dependency_not_ready"
|
||||
return "route_source_present_public_edge_or_route_reprobe_required"
|
||||
|
||||
|
||||
def _active_blockers(
|
||||
*,
|
||||
runtime: dict[str, Any],
|
||||
route_contract: dict[str, Any],
|
||||
candidate_packaged: bool,
|
||||
runtime_ready: bool,
|
||||
) -> list[str]:
|
||||
if runtime_ready:
|
||||
return []
|
||||
|
||||
blockers = _strings(runtime.get("active_blockers"))
|
||||
if route_contract.get("route_target_ready") is not True:
|
||||
blockers.append("stockplatform_route_target_selector_missing")
|
||||
if candidate_packaged:
|
||||
blockers.append("stockplatform_110_control_path_readback_required")
|
||||
return _unique_strings(blockers)
|
||||
|
||||
|
||||
def _current_apply_blocker(
|
||||
*,
|
||||
runtime_ready: bool,
|
||||
candidate_packaged: bool,
|
||||
api_layer_classification: str,
|
||||
) -> str:
|
||||
if runtime_ready:
|
||||
return ""
|
||||
if not candidate_packaged:
|
||||
return "route_or_public_edge_evidence_required"
|
||||
if api_layer_classification == "api_live_data_dependency_not_ready":
|
||||
return (
|
||||
"data_dependency_control_path_readback_required_before_source_"
|
||||
"freshness_or_postgres_contract_apply"
|
||||
)
|
||||
return "host_control_path_readback_required_before_targeted_api_reconcile"
|
||||
|
||||
|
||||
def _allowed_scope(api_layer_classification: str) -> list[str]:
|
||||
common = [
|
||||
"public_api_post_apply_verifier",
|
||||
"km_rag_mcp_playbook_metadata_writeback",
|
||||
]
|
||||
if api_layer_classification == "api_live_data_dependency_not_ready":
|
||||
return [
|
||||
"read_only_data_dependency_status_inspection",
|
||||
"read_only_source_freshness_contract_diff",
|
||||
"read_only_postgres_connection_contract_readback",
|
||||
"controlled_metadata_or_config_contract_apply_after_dry_run",
|
||||
*common,
|
||||
]
|
||||
return [
|
||||
"read_only_container_status_inspection",
|
||||
"read_only_redacted_api_log_tail",
|
||||
"targeted_api_container_reconcile_after_failed_health_check",
|
||||
*common,
|
||||
]
|
||||
|
||||
|
||||
def _status(
|
||||
*,
|
||||
runtime_ready: bool,
|
||||
candidate_packaged: bool,
|
||||
route_target_ready: bool,
|
||||
api_layer_classification: str,
|
||||
) -> str:
|
||||
if runtime_ready:
|
||||
return "stockplatform_public_api_recovery_preflight_not_required"
|
||||
if (
|
||||
candidate_packaged
|
||||
and api_layer_classification == "api_live_data_dependency_not_ready"
|
||||
):
|
||||
return "controlled_data_dependency_preflight_packaged_waiting_control_path_readback"
|
||||
if candidate_packaged:
|
||||
return "controlled_recovery_preflight_packaged_waiting_control_path_readback"
|
||||
if route_target_ready:
|
||||
return "blocked_public_edge_or_route_symptom_requires_reprobe"
|
||||
return "blocked_route_target_selector_missing"
|
||||
|
||||
|
||||
def _safe_next_step(
|
||||
*,
|
||||
runtime_ready: bool,
|
||||
candidate_packaged: bool,
|
||||
route_target_ready: bool,
|
||||
api_layer_classification: str,
|
||||
) -> str:
|
||||
if runtime_ready:
|
||||
return "stockplatform_public_api_ready_keep_p0_006_reboot_window_gate"
|
||||
if (
|
||||
candidate_packaged
|
||||
and api_layer_classification == "api_live_data_dependency_not_ready"
|
||||
):
|
||||
return (
|
||||
"run_stockplatform_data_dependency_readback_then_source_freshness_"
|
||||
"and_postgres_contract_preflight_no_db_write"
|
||||
)
|
||||
if candidate_packaged:
|
||||
return (
|
||||
"restore_stockplatform_110_control_path_readback_then_run_read_only_"
|
||||
"container_inspection_and_targeted_api_reconcile_with_public_verifier"
|
||||
)
|
||||
if route_target_ready:
|
||||
return "rerun_public_api_readback_and_188_to_110_upstream_probe_before_apply"
|
||||
return "repair_source_route_target_selector_before_any_runtime_recovery"
|
||||
|
||||
|
||||
def _controlled_work_items(
|
||||
*,
|
||||
candidate_packaged: bool,
|
||||
runtime_ready: bool,
|
||||
api_layer_classification: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
if runtime_ready:
|
||||
return []
|
||||
base_status = "queued_waiting_control_path_readback" if candidate_packaged else "blocked_waiting_route_evidence"
|
||||
if api_layer_classification == "api_live_data_dependency_not_ready":
|
||||
return [
|
||||
{
|
||||
"id": "stockplatform-data-control-path-readback",
|
||||
"title": "StockPlatform data dependency control path readback",
|
||||
"owner_agent": "DevOps Agent",
|
||||
"status": base_status,
|
||||
"required_output": "non_secret_host_liveness_and_runtime_contract_match",
|
||||
},
|
||||
{
|
||||
"id": "stockplatform-data-dependency-inspection",
|
||||
"title": "Read-only freshness / ingestion dependency inspection",
|
||||
"owner_agent": "SRE Agent",
|
||||
"status": "waiting_control_path_readback",
|
||||
"required_output": "freshness_ingestion_source_and_postgres_contract_status",
|
||||
},
|
||||
{
|
||||
"id": "stockplatform-data-contract-preflight",
|
||||
"title": "Source freshness and Postgres contract preflight",
|
||||
"owner_agent": "Data Reliability Agent",
|
||||
"status": "waiting_read_only_inspection",
|
||||
"required_output": "dry_run_contract_diff_no_db_write",
|
||||
},
|
||||
{
|
||||
"id": "stockplatform-api-post-apply-verifier",
|
||||
"title": "Public API post-apply verifier",
|
||||
"owner_agent": "Verifier Agent",
|
||||
"status": "ready",
|
||||
"required_output": "public_health_freshness_ingestion_green",
|
||||
},
|
||||
{
|
||||
"id": "stockplatform-data-learning-writeback",
|
||||
"title": "KM / RAG / MCP / PlayBook trust writeback",
|
||||
"owner_agent": "Learning Agent",
|
||||
"status": "ready",
|
||||
"required_output": "metadata_only_data_dependency_receipt_and_trust_delta",
|
||||
},
|
||||
]
|
||||
return [
|
||||
{
|
||||
"id": "stockplatform-api-control-path-readback",
|
||||
"title": "StockPlatform host control path readback",
|
||||
"owner_agent": "DevOps Agent",
|
||||
"status": base_status,
|
||||
"required_output": "non_secret_host_liveness_and_compose_project_match",
|
||||
},
|
||||
{
|
||||
"id": "stockplatform-api-runtime-inspection",
|
||||
"title": "Read-only API container status inspection",
|
||||
"owner_agent": "SRE Agent",
|
||||
"status": "waiting_control_path_readback",
|
||||
"required_output": "api_container_state_and_redacted_health_log_tail",
|
||||
},
|
||||
{
|
||||
"id": "stockplatform-api-targeted-reconcile",
|
||||
"title": "Targeted API container reconcile",
|
||||
"owner_agent": "DevOps Agent",
|
||||
"status": "waiting_read_only_inspection",
|
||||
"required_output": "bounded_api_reconcile_receipt_no_daemon_restart",
|
||||
},
|
||||
{
|
||||
"id": "stockplatform-api-post-apply-verifier",
|
||||
"title": "Public API post-apply verifier",
|
||||
"owner_agent": "Verifier Agent",
|
||||
"status": "ready",
|
||||
"required_output": "public_health_freshness_ingestion_green",
|
||||
},
|
||||
{
|
||||
"id": "stockplatform-api-learning-writeback",
|
||||
"title": "KM / RAG / MCP / PlayBook trust writeback",
|
||||
"owner_agent": "Learning Agent",
|
||||
"status": "ready",
|
||||
"required_output": "metadata_only_recovery_receipt_and_trust_delta",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _controlled_action_candidates(
|
||||
*,
|
||||
candidate_packaged: bool,
|
||||
runtime_ready: bool,
|
||||
api_layer_classification: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
if runtime_ready:
|
||||
return []
|
||||
if api_layer_classification == "api_live_data_dependency_not_ready":
|
||||
return [
|
||||
{
|
||||
"id": "read_only_data_dependency_inspection",
|
||||
"execution_route": "stockplatform_host_controlled_runtime_executor",
|
||||
"current_status": (
|
||||
"queued_waiting_control_path_readback"
|
||||
if candidate_packaged
|
||||
else "blocked_waiting_route_evidence"
|
||||
),
|
||||
"check_mode": "read_only_freshness_ingestion_and_postgres_contract",
|
||||
"apply_allowed_when": [
|
||||
"host_control_path_liveness_readback_ok",
|
||||
"stockplatform_api_health_ok",
|
||||
"source_freshness_contract_diff_present",
|
||||
"post_apply_public_verifier_ready",
|
||||
],
|
||||
"forbidden_actions": [
|
||||
"secret_read",
|
||||
"docker_daemon_restart",
|
||||
"host_reboot",
|
||||
"database_write",
|
||||
"fake_freshness_marker",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "source_freshness_contract_preflight",
|
||||
"execution_route": "stockplatform_data_contract_executor",
|
||||
"current_status": "waiting_read_only_data_dependency_inspection",
|
||||
"check_mode": "dry_run_source_and_runtime_contract_diff",
|
||||
"bounded_apply": (
|
||||
"apply only metadata/config contract repair after dry-run; "
|
||||
"no DB write, no fake freshness marker, no destructive restore"
|
||||
),
|
||||
},
|
||||
]
|
||||
return [
|
||||
{
|
||||
"id": "read_only_container_inspection",
|
||||
"execution_route": "stockplatform_host_controlled_runtime_executor",
|
||||
"current_status": (
|
||||
"queued_waiting_control_path_readback"
|
||||
if candidate_packaged
|
||||
else "blocked_waiting_route_evidence"
|
||||
),
|
||||
"check_mode": "read_only_status_and_redacted_logs",
|
||||
"apply_allowed_when": [
|
||||
"host_control_path_liveness_readback_ok",
|
||||
"compose_project_or_container_target_matches_stockplatform_api",
|
||||
"secret_scrubber_enabled_for_log_tail",
|
||||
],
|
||||
"forbidden_actions": [
|
||||
"secret_read",
|
||||
"docker_daemon_restart",
|
||||
"host_reboot",
|
||||
"database_write",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "targeted_api_container_reconcile",
|
||||
"execution_route": "stockplatform_host_controlled_runtime_executor",
|
||||
"current_status": "waiting_read_only_container_inspection",
|
||||
"check_mode": "dry_run_target_selector_and_current_state_diff",
|
||||
"apply_allowed_when": [
|
||||
"api_container_missing_exited_or_unhealthy",
|
||||
"current_image_and_compose_source_match_expected_runtime",
|
||||
"post_apply_public_verifier_ready",
|
||||
"rollback_receipt_path_ready",
|
||||
],
|
||||
"bounded_apply": (
|
||||
"reconcile only the StockPlatform API container/service target; "
|
||||
"no Docker daemon restart, no host reboot, no DB write"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _post_apply_verifiers(*, runtime_ready: bool) -> list[dict[str, Any]]:
|
||||
if runtime_ready:
|
||||
return []
|
||||
return [
|
||||
{
|
||||
"id": "stockplatform_public_healthz",
|
||||
"url": "https://stock.wooo.work/api/healthz",
|
||||
"expected_http_status": 200,
|
||||
},
|
||||
{
|
||||
"id": "stockplatform_freshness",
|
||||
"url": "https://stock.wooo.work/api/v1/system/freshness",
|
||||
"expected_status": "ok",
|
||||
},
|
||||
{
|
||||
"id": "stockplatform_ingestion",
|
||||
"url": "https://stock.wooo.work/api/v1/system/ingestion",
|
||||
"expected_status": "ok",
|
||||
},
|
||||
{
|
||||
"id": "awoooi_priority_work_order_readback",
|
||||
"url": "/api/v1/agents/awoooi-priority-work-order-readback",
|
||||
"expected_rollup": "stockplatform_public_api_runtime_ready=true",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _learning_writeback_contracts() -> list[dict[str, Any]]:
|
||||
targets = [
|
||||
"timeline_event",
|
||||
"knowledge_entry",
|
||||
"rag_embedding",
|
||||
"mcp_tool_registry_signal",
|
||||
"playbook_trust_score",
|
||||
"controlled_recovery_report",
|
||||
]
|
||||
return [
|
||||
{
|
||||
"target": target,
|
||||
"writeback_mode": "metadata_only",
|
||||
"raw_log_allowed": False,
|
||||
"secret_value_allowed": False,
|
||||
"receipt_key": f"stockplatform_public_api_recovery::{target}",
|
||||
}
|
||||
for target in targets
|
||||
]
|
||||
|
||||
|
||||
def _dict(value: Any) -> dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _int(value: Any) -> int:
|
||||
try:
|
||||
return int(str(value))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _strings(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [str(item) for item in value if item is not None]
|
||||
|
||||
|
||||
def _unique_strings(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
unique: list[str] = []
|
||||
for value in values:
|
||||
if value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
unique.append(value)
|
||||
return unique
|
||||
@@ -10,9 +10,13 @@ from fastapi.testclient import TestClient
|
||||
from src.api.v1 import agents
|
||||
from src.api.v1.agents import router
|
||||
from src.services.awoooi_priority_work_order_readback import (
|
||||
apply_stockplatform_public_api_controlled_recovery_preflight,
|
||||
apply_stockplatform_public_api_runtime_readback,
|
||||
load_latest_awoooi_priority_work_order_readback,
|
||||
)
|
||||
from src.services.stockplatform_public_api_controlled_recovery_preflight import (
|
||||
load_latest_stockplatform_public_api_controlled_recovery_preflight,
|
||||
)
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
_SNAPSHOT_PATH = (
|
||||
@@ -135,6 +139,77 @@ def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift()
|
||||
)
|
||||
|
||||
|
||||
def test_awoooi_priority_work_order_readback_overlays_controlled_recovery_preflight():
|
||||
payload = load_latest_awoooi_priority_work_order_readback()
|
||||
runtime = _stockplatform_runtime_blocked()
|
||||
preflight = load_latest_stockplatform_public_api_controlled_recovery_preflight(
|
||||
runtime_readback=runtime
|
||||
)
|
||||
|
||||
apply_stockplatform_public_api_runtime_readback(payload, runtime)
|
||||
apply_stockplatform_public_api_controlled_recovery_preflight(payload, preflight)
|
||||
|
||||
state = payload["mainline_execution_state"]
|
||||
in_progress = payload["in_progress_or_blocked_in_priority_order"][0]
|
||||
evidence = in_progress["evidence"]
|
||||
assert payload["status"] == (
|
||||
"p0_006_blocked_stockplatform_public_api_controlled_recovery_preflight"
|
||||
)
|
||||
assert state["active_p0_state"] == (
|
||||
"blocked_stockplatform_public_api_controlled_recovery_preflight"
|
||||
)
|
||||
assert state["next_executable_mainline_workplan_id"] == (
|
||||
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT"
|
||||
)
|
||||
assert evidence["stockplatform_controlled_recovery_preflight_status"] == (
|
||||
"controlled_recovery_preflight_packaged_waiting_control_path_readback"
|
||||
)
|
||||
assert evidence["stockplatform_controlled_recovery_candidate_packaged"] is True
|
||||
assert evidence["stockplatform_host_control_path_readback_required"] is True
|
||||
assert evidence["stockplatform_controlled_work_item_count"] == 5
|
||||
assert evidence["stockplatform_post_apply_verifier_count"] == 4
|
||||
assert evidence["stockplatform_learning_writeback_contract_count"] == 6
|
||||
assert evidence["stockplatform_current_apply_allowed"] is False
|
||||
assert evidence["stockplatform_runtime_write_gate"] == "controlled_after_preflight"
|
||||
assert "not a manual end state" in in_progress["reason"]
|
||||
assert "Do not restart Docker daemon" in in_progress["professional_fix"]["action"]
|
||||
assert payload["summary"][
|
||||
"stockplatform_public_api_controlled_recovery_candidate_packaged"
|
||||
] is True
|
||||
assert payload["next_execution_order"][0].startswith(
|
||||
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT"
|
||||
)
|
||||
|
||||
|
||||
def test_awoooi_priority_work_order_readback_routes_stockplatform_data_dependency():
|
||||
payload = load_latest_awoooi_priority_work_order_readback()
|
||||
runtime = _stockplatform_runtime_data_dependency_blocked()
|
||||
preflight = load_latest_stockplatform_public_api_controlled_recovery_preflight(
|
||||
runtime_readback=runtime
|
||||
)
|
||||
|
||||
apply_stockplatform_public_api_runtime_readback(payload, runtime)
|
||||
apply_stockplatform_public_api_controlled_recovery_preflight(payload, preflight)
|
||||
|
||||
state = payload["mainline_execution_state"]
|
||||
in_progress = payload["in_progress_or_blocked_in_priority_order"][0]
|
||||
evidence = in_progress["evidence"]
|
||||
assert evidence["stockplatform_controlled_recovery_preflight_status"] == (
|
||||
"controlled_data_dependency_preflight_packaged_waiting_control_path_readback"
|
||||
)
|
||||
assert evidence["stockplatform_controlled_recovery_api_layer"] == (
|
||||
"api_live_data_dependency_not_ready"
|
||||
)
|
||||
assert state["next_executable_mainline_state"] == (
|
||||
"controlled_recovery_preflight_waiting_stockplatform_data_dependency_"
|
||||
"readback_and_source_freshness_contract"
|
||||
)
|
||||
assert "data dependency/source freshness" in in_progress["reason"]
|
||||
assert "Do not restart Docker daemon" in in_progress["professional_fix"]["action"]
|
||||
assert "write DB rows" in in_progress["professional_fix"]["action"]
|
||||
assert "do not DB-write or fake freshness" in payload["next_execution_order"][0]
|
||||
|
||||
|
||||
def test_awoooi_priority_work_order_readback_normalizes_runtime_source_truth(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
@@ -230,6 +305,7 @@ def _stockplatform_runtime_blocked() -> dict:
|
||||
"runtime_ready": False,
|
||||
"live_drift_from_committed_scorecard": True,
|
||||
"readback": {
|
||||
"web_health_http_status": 200,
|
||||
"api_health_http_status": 502,
|
||||
"freshness_http_status": 502,
|
||||
"ingestion_http_status": 502,
|
||||
@@ -245,3 +321,35 @@ def _stockplatform_runtime_blocked() -> dict:
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _stockplatform_runtime_data_dependency_blocked() -> dict:
|
||||
return {
|
||||
"schema_version": "stockplatform_public_api_runtime_readback_v1",
|
||||
"status": "blocked_stockplatform_public_api_runtime_drift",
|
||||
"safe_next_step": "run_stockplatform_data_dependency_controlled_preflight",
|
||||
"active_blockers": [
|
||||
"stockplatform_freshness_postgres_not_ready",
|
||||
"stockplatform_ingestion_postgres_not_ready",
|
||||
"stockplatform_recovery_control_path_receipt_missing",
|
||||
"stockplatform_controlled_deploy_or_ssh_readback_required",
|
||||
],
|
||||
"runtime_ready": False,
|
||||
"live_drift_from_committed_scorecard": True,
|
||||
"readback": {
|
||||
"web_health_http_status": 200,
|
||||
"api_health_http_status": 200,
|
||||
"freshness_http_status": 200,
|
||||
"ingestion_http_status": 200,
|
||||
},
|
||||
"rollups": {
|
||||
"http_502_count": 0,
|
||||
},
|
||||
"recovery_control_path": {
|
||||
"status": "blocked_recovery_control_path_receipt_missing",
|
||||
"active_blockers": [
|
||||
"stockplatform_recovery_control_path_receipt_missing",
|
||||
"stockplatform_controlled_deploy_or_ssh_readback_required",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.api.v1 import agents
|
||||
from src.api.v1.agents import router
|
||||
from src.services.stockplatform_public_api_controlled_recovery_preflight import (
|
||||
load_latest_stockplatform_public_api_controlled_recovery_preflight,
|
||||
)
|
||||
|
||||
|
||||
def test_stockplatform_controlled_recovery_preflight_packages_api_502(
|
||||
tmp_path: Path,
|
||||
):
|
||||
payload = load_latest_stockplatform_public_api_controlled_recovery_preflight(
|
||||
runtime_readback=_runtime_blocked(),
|
||||
route_source_path=_route_source(tmp_path),
|
||||
)
|
||||
|
||||
assert (
|
||||
payload["schema_version"]
|
||||
== "stockplatform_public_api_controlled_recovery_preflight_v1"
|
||||
)
|
||||
assert payload["priority"] == "P0-006"
|
||||
assert payload["status"] == (
|
||||
"controlled_recovery_preflight_packaged_waiting_control_path_readback"
|
||||
)
|
||||
assert payload["api_layer_classification"] == (
|
||||
"edge_alive_api_upstream_unavailable"
|
||||
)
|
||||
assert payload["target_selector"]["route_target_ready"] is True
|
||||
assert payload["source_of_truth_diff"]["status"] == (
|
||||
"route_source_present_runtime_api_upstream_blocked"
|
||||
)
|
||||
assert (
|
||||
payload["controlled_apply_policy"]["runtime_write_gate"]
|
||||
== "controlled_after_preflight"
|
||||
)
|
||||
assert payload["controlled_apply_policy"]["break_glass_required"] is False
|
||||
assert payload["controlled_apply_policy"]["current_apply_allowed"] is False
|
||||
assert "stockplatform_110_control_path_readback_required" in payload[
|
||||
"active_blockers"
|
||||
]
|
||||
assert payload["rollups"]["controlled_recovery_candidate_packaged"] is True
|
||||
assert payload["rollups"]["host_control_path_readback_required"] is True
|
||||
assert payload["rollups"]["controlled_work_item_count"] == 5
|
||||
assert payload["rollups"]["post_apply_verifier_count"] == 4
|
||||
assert payload["rollups"]["learning_writeback_contract_count"] == 6
|
||||
assert payload["controlled_work_items"][0]["status"] == (
|
||||
"queued_waiting_control_path_readback"
|
||||
)
|
||||
assert payload["post_apply_verifiers"][0]["expected_http_status"] == 200
|
||||
assert payload["learning_writeback_contracts"][0]["writeback_mode"] == (
|
||||
"metadata_only"
|
||||
)
|
||||
assert payload["operation_boundaries"]["ssh_used"] is False
|
||||
assert payload["operation_boundaries"]["docker_command_performed"] is False
|
||||
assert payload["operation_boundaries"]["docker_daemon_restart_performed"] is False
|
||||
assert payload["operation_boundaries"]["host_reboot_performed"] is False
|
||||
assert payload["operation_boundaries"]["database_write_or_restore_performed"] is False
|
||||
assert payload["operation_boundaries"]["secret_value_collection_allowed"] is False
|
||||
|
||||
|
||||
def test_stockplatform_controlled_recovery_preflight_not_required_when_ready(
|
||||
tmp_path: Path,
|
||||
):
|
||||
payload = load_latest_stockplatform_public_api_controlled_recovery_preflight(
|
||||
runtime_readback=_runtime_ready(),
|
||||
route_source_path=_route_source(tmp_path),
|
||||
)
|
||||
|
||||
assert payload["status"] == "stockplatform_public_api_recovery_preflight_not_required"
|
||||
assert payload["active_blockers"] == []
|
||||
assert payload["controlled_work_items"] == []
|
||||
assert payload["rollups"]["recovery_required"] is False
|
||||
|
||||
|
||||
def test_stockplatform_controlled_recovery_preflight_packages_data_dependency(
|
||||
tmp_path: Path,
|
||||
):
|
||||
payload = load_latest_stockplatform_public_api_controlled_recovery_preflight(
|
||||
runtime_readback=_runtime_data_dependency_blocked(),
|
||||
route_source_path=_route_source(tmp_path),
|
||||
)
|
||||
|
||||
assert payload["status"] == (
|
||||
"controlled_data_dependency_preflight_packaged_waiting_control_path_readback"
|
||||
)
|
||||
assert payload["api_layer_classification"] == "api_live_data_dependency_not_ready"
|
||||
assert payload["source_of_truth_diff"]["status"] == (
|
||||
"route_source_present_api_live_data_dependency_not_ready"
|
||||
)
|
||||
assert payload["controlled_apply_policy"]["current_apply_blocker"] == (
|
||||
"data_dependency_control_path_readback_required_before_source_"
|
||||
"freshness_or_postgres_contract_apply"
|
||||
)
|
||||
assert "read_only_data_dependency_status_inspection" in payload[
|
||||
"controlled_apply_policy"
|
||||
]["allowed_scope"]
|
||||
assert "fake_freshness_marker" in payload["controlled_action_candidates"][0][
|
||||
"forbidden_actions"
|
||||
]
|
||||
assert payload["controlled_work_items"][1]["id"] == (
|
||||
"stockplatform-data-dependency-inspection"
|
||||
)
|
||||
assert payload["controlled_action_candidates"][0]["id"] == (
|
||||
"read_only_data_dependency_inspection"
|
||||
)
|
||||
assert payload["rollups"]["controlled_recovery_candidate_packaged"] is True
|
||||
assert payload["operation_boundaries"]["database_write_or_restore_performed"] is False
|
||||
|
||||
|
||||
def test_stockplatform_controlled_recovery_preflight_endpoint_returns_payload(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
agents,
|
||||
"load_latest_stockplatform_public_api_controlled_recovery_preflight",
|
||||
lambda: load_latest_stockplatform_public_api_controlled_recovery_preflight(
|
||||
runtime_readback=_runtime_blocked(),
|
||||
route_source_path=_route_source(tmp_path),
|
||||
),
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api/v1")
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get(
|
||||
"/api/v1/agents/stockplatform-public-api-controlled-recovery-preflight"
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == (
|
||||
"controlled_recovery_preflight_packaged_waiting_control_path_readback"
|
||||
)
|
||||
assert data["rollups"]["controlled_recovery_candidate_packaged"] is True
|
||||
|
||||
|
||||
def _route_source(tmp_path: Path) -> Path:
|
||||
path = tmp_path / "188-all-sites.conf.j2"
|
||||
path.write_text(
|
||||
"""
|
||||
# Stock Platform - stock.wooo.work
|
||||
server {
|
||||
server_name stock.wooo.work;
|
||||
location / {
|
||||
proxy_pass http://192.168.0.110:31235;
|
||||
}
|
||||
}
|
||||
# Next Site
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def _runtime_blocked() -> dict:
|
||||
return {
|
||||
"schema_version": "stockplatform_public_api_runtime_readback_v1",
|
||||
"priority": "P0-006",
|
||||
"status": "blocked_stockplatform_public_api_runtime_drift",
|
||||
"safe_next_step": (
|
||||
"recover_stockplatform_api_container_runtime_or_docker_control_path_"
|
||||
"without_daemon_restart_then_rerun_public_api_readback"
|
||||
),
|
||||
"active_blockers": [
|
||||
"stockplatform_public_api_healthz_http_502",
|
||||
"stockplatform_freshness_http_502",
|
||||
"stockplatform_ingestion_http_502",
|
||||
],
|
||||
"runtime_ready": False,
|
||||
"base_url": "https://stock.wooo.work",
|
||||
"readback": {
|
||||
"web_health_http_status": 200,
|
||||
"api_health_http_status": 502,
|
||||
"freshness_http_status": 502,
|
||||
"ingestion_http_status": 502,
|
||||
},
|
||||
"rollups": {
|
||||
"http_502_count": 3,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _runtime_ready() -> dict:
|
||||
return {
|
||||
"schema_version": "stockplatform_public_api_runtime_readback_v1",
|
||||
"priority": "P0-006",
|
||||
"status": "stockplatform_public_api_runtime_ready",
|
||||
"runtime_ready": True,
|
||||
"base_url": "https://stock.wooo.work",
|
||||
"active_blockers": [],
|
||||
"readback": {
|
||||
"web_health_http_status": 200,
|
||||
"api_health_http_status": 200,
|
||||
"freshness_http_status": 200,
|
||||
"ingestion_http_status": 200,
|
||||
},
|
||||
"rollups": {
|
||||
"http_502_count": 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _runtime_data_dependency_blocked() -> dict:
|
||||
return {
|
||||
"schema_version": "stockplatform_public_api_runtime_readback_v1",
|
||||
"priority": "P0-006",
|
||||
"status": "blocked_stockplatform_public_api_runtime_drift",
|
||||
"safe_next_step": "run_stockplatform_data_dependency_controlled_preflight",
|
||||
"active_blockers": [
|
||||
"stockplatform_freshness_postgres_not_ready",
|
||||
"stockplatform_ingestion_postgres_not_ready",
|
||||
"stockplatform_recovery_control_path_receipt_missing",
|
||||
"stockplatform_controlled_deploy_or_ssh_readback_required",
|
||||
],
|
||||
"runtime_ready": False,
|
||||
"base_url": "https://stock.wooo.work",
|
||||
"readback": {
|
||||
"web_health_http_status": 200,
|
||||
"api_health_http_status": 200,
|
||||
"freshness_http_status": 200,
|
||||
"ingestion_http_status": 200,
|
||||
},
|
||||
"rollups": {
|
||||
"http_502_count": 0,
|
||||
},
|
||||
}
|
||||
@@ -50299,3 +50299,19 @@ production browser smoke:
|
||||
- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth。
|
||||
- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。
|
||||
- 沒有重啟主機,沒有 workflow_dispatch,沒有 host / Docker / K8s / DB / firewall runtime 寫入。
|
||||
|
||||
## 2026-06-30 — 16:45 P0-006 StockPlatform API controlled recovery preflight
|
||||
|
||||
**完成內容**:
|
||||
- 新增 `stockplatform_public_api_controlled_recovery_preflight`,把 StockPlatform public API 502 從單純 readback 推進為 AI controlled recovery preflight;輸出 target selector、source-of-truth diff、API layer classification、controlled action candidates、post-apply verifiers 與 KM / RAG / MCP / PlayBook metadata writeback contracts。
|
||||
- 新增 production route `GET /api/v1/agents/stockplatform-public-api-controlled-recovery-preflight`;端點只讀 public HTTPS readback 與 repo 內 188 Nginx source-of-truth,不 SSH、不 Docker、不 restart、不寫 DB、不讀 secret、不觸發 workflow。
|
||||
- `awoooi_priority_work_order_readback` 新增 recovery preflight overlay;當 StockPlatform public web 200 但 public API/freshness/ingestion 502 時,下一個主線工作會前進到 `P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT`,不是停在人工終局。
|
||||
- Gitea CD controlled-runtime 白名單補入新 preflight service / test,避免 P0-006 窄 API 變更誤跑 full/B5 profile。
|
||||
|
||||
**受控邊界**:
|
||||
- 允許候選:恢復/驗證 110 control path readback、read-only API container inspection、redacted log tail、必要時僅針對 StockPlatform API container 做 bounded reconcile,並以 public API health/freshness/ingestion + priority readback 驗證。
|
||||
- 禁止事項維持:Docker daemon restart、host reboot、Nginx reload/restart、DB write/restore/prune、secret value read、workflow dispatch、force push / repo refs mutation。
|
||||
|
||||
**仍需驗證**:
|
||||
- 跑 focused API tests、py_compile、ruff、Gitea runner pressure guard、Gitea secret step guard 與 `git diff --check`。
|
||||
- 推送後等待 deploy marker,再讀回 priority endpoint 與新 preflight endpoint,確認 production 顯示 recovery preflight 而非 manual/default-human 終局。
|
||||
|
||||
@@ -320,13 +320,17 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() -
|
||||
"apps/api/src/services/reboot_auto_recovery_slo_scorecard.py)",
|
||||
"apps/api/src/services/reboot_auto_recovery_drill_preflight.py)",
|
||||
"apps/api/src/services/stockplatform_public_api_runtime_readback.py)",
|
||||
"apps/api/src/services/stockplatform_public_api_controlled_recovery_preflight.py)",
|
||||
"apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py)",
|
||||
"apps/api/tests/test_stockplatform_public_api_runtime_readback.py)",
|
||||
"apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py)",
|
||||
"src/services/reboot_auto_recovery_slo_scorecard.py",
|
||||
"src/services/reboot_auto_recovery_drill_preflight.py",
|
||||
"src/services/stockplatform_public_api_runtime_readback.py",
|
||||
"src/services/stockplatform_public_api_controlled_recovery_preflight.py",
|
||||
"tests/test_reboot_auto_recovery_slo_scorecard_api.py",
|
||||
"tests/test_stockplatform_public_api_runtime_readback.py",
|
||||
"tests/test_stockplatform_public_api_controlled_recovery_preflight.py",
|
||||
"scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service)",
|
||||
"scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer)",
|
||||
"scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh)",
|
||||
|
||||
Reference in New Issue
Block a user