wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(iwooos): show controlled cd lane live metric proof
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Failing after 42s
Details
CD Pipeline / build-and-deploy (push) Has been skipped
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details

Browse Source
This commit is contained in:
Your Name
2026-07-02 12:46:51 +08:00
parent f426610d83
commit e7cf3f2aad
6 changed files with 508 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/src/api/v1/agents.py
View File

@@ -340,9 +340,11 @@ from src.services.awoooi_gitea_onboarding_warning_step_template_copy_receipt imp
)
from src.services.awoooi_priority_work_order_readback import (
apply_ai_loop_current_blocker_execution_queue,
apply_controlled_cd_lane_live_metric_readback,
apply_harbor_registry_controlled_recovery_preflight,
apply_stockplatform_public_api_controlled_recovery_preflight,
apply_stockplatform_public_api_runtime_readback,
load_controlled_cd_lane_live_metric_readback,
load_latest_awoooi_priority_work_order_readback,
)
from src.services.awoooi_status_cleanup_dashboard import (
@@ -1089,7 +1091,8 @@ async def get_delivery_closure_workbench() -> dict[str, Any]:
"讀取已提交的 AWOOOI P0/P1 主線工作順序快照；此端點只回傳"
"目前 active P0、已關閉 P0、下一步順序、禁止事項與 evidence refs。"
"它會讀取 StockPlatform public HTTPS health/freshness/ingestion live "
"readback；不讀 raw sessions / SQLite、不呼叫 GitHub / Gitea live API、"
"readback，並讀取 110 node_exporter 的 controlled CD lane textfile metrics；"
"不讀 raw sessions / SQLite、不呼叫 GitHub / Gitea live API、不 SSH、"
"不讀 secret、不註冊 runner、不觸發 workflow、不操作 host / Docker / K8s / DB / firewall。"
),
)
@@ -1122,6 +1125,13 @@ async def get_awoooi_priority_work_order_readback() -> dict[str, Any]:
load_latest_ai_agent_log_controlled_writeback_executor_readback
)
apply_ai_loop_current_blocker_execution_queue(payload, log_executor)
controlled_cd_lane_metrics = await asyncio.to_thread(
load_controlled_cd_lane_live_metric_readback
)
apply_controlled_cd_lane_live_metric_readback(
payload,
controlled_cd_lane_metrics,
)
return redact_public_lan_topology(payload)
except FileNotFoundError as exc:
raise HTTPException(

233
apps/api/src/services/awoooi_priority_work_order_readback.py
View File

@@ -9,6 +9,8 @@ from __future__ import annotations
import json
import re
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path
from typing import Any
@@ -19,7 +21,15 @@ from src.services.snapshot_paths import default_operations_dir
_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__))
_SNAPSHOT_FILE = "awoooi-priority-work-order-readback.snapshot.json"
_SCHEMA_VERSION = "awoooi_priority_work_order_readback_v1"
_CONTROLLED_CD_LANE_LIVE_METRIC_SCHEMA_VERSION = (
"controlled_cd_lane_live_metric_readback_v1"
)
_CONTROLLED_CD_LANE_NODE_EXPORTER_URL = "http://192.168.0.110:9100/metrics"
_SHA_RE = re.compile(r"^[0-9a-f]{40}$")
_METRIC_LINE_RE = re.compile(
r"^(?P<name>[a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{[^}]*\})?\s+"
r"(?P<value>[-+]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][-+]?[0-9]+)?)$"
)
def load_latest_awoooi_priority_work_order_readback(
@@ -41,6 +51,141 @@ def load_latest_awoooi_priority_work_order_readback(
return payload
def load_controlled_cd_lane_live_metric_readback(
metrics_url: str = _CONTROLLED_CD_LANE_NODE_EXPORTER_URL,
timeout_seconds: float = 3.0,
) -> dict[str, Any]:
"""Read 110 node_exporter text metrics for the controlled CD lane.
This is a read-only HTTP metrics probe. It does not SSH, read runner
registration contents, touch Docker/systemd, or perform runtime writes.
"""
try:
request = urllib.request.Request(metrics_url, method="GET")
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
raw = response.read(2_000_000).decode("utf-8", errors="replace")
except (OSError, TimeoutError, urllib.error.URLError) as exc:
return {
"schema_version": _CONTROLLED_CD_LANE_LIVE_METRIC_SCHEMA_VERSION,
"status": "controlled_cd_lane_live_metric_unavailable",
"source": "node_exporter_text_metrics",
"source_url": metrics_url,
"metadata_only": True,
"active_blockers": ["controlled_cd_lane_live_metric_unavailable"],
"error_class": exc.__class__.__name__,
"readback": {
"enforcer_metric_present": False,
"controlled_drain_active_allowed": False,
"controlled_drain_staging_allowed": False,
"enforcer_last_run_fresh": False,
},
}
metrics = _parse_prometheus_text_metrics(raw)
readback = _build_controlled_cd_lane_metric_readback(metrics)
active_blockers: list[str] = []
if not readback["enforcer_metric_present"]:
active_blockers.append("controlled_cd_lane_enforcer_metric_missing")
if not readback["enforcer_last_run_fresh"]:
active_blockers.append("controlled_cd_lane_enforcer_metric_stale")
if not readback["controlled_drain_active_allowed"]:
active_blockers.append("controlled_cd_lane_active_not_allowed_yet")
if not readback["controlled_drain_staging_allowed"]:
active_blockers.append("controlled_cd_lane_staging_not_allowed_yet")
return {
"schema_version": _CONTROLLED_CD_LANE_LIVE_METRIC_SCHEMA_VERSION,
"status": (
"controlled_cd_lane_live_metric_readback_ready"
if readback["enforcer_metric_present"]
else "controlled_cd_lane_live_metric_missing"
),
"source": "node_exporter_text_metrics",
"source_url": metrics_url,
"metadata_only": True,
"active_blockers": active_blockers,
"readback": readback,
}
def apply_controlled_cd_lane_live_metric_readback(
payload: dict[str, Any],
live_metric_readback: dict[str, Any],
) -> None:
"""Overlay live controlled CD lane metric truth onto P0-006 readback."""
readback = _dict(live_metric_readback.get("readback"))
state = _dict(payload.setdefault("mainline_execution_state", {}))
active_blockers = _strings(live_metric_readback.get("active_blockers"))
metric_fields = {
"controlled_cd_lane_live_metric_status": str(
live_metric_readback.get("status") or "unknown"
),
"controlled_cd_lane_live_metric_source": str(
live_metric_readback.get("source") or "node_exporter_text_metrics"
),
"controlled_cd_lane_live_metric_metadata_only": bool(
live_metric_readback.get("metadata_only") is True
),
"controlled_cd_lane_live_enforcer_metric_present": bool(
readback.get("enforcer_metric_present") is True
),
"controlled_cd_lane_live_enforcer_last_run_timestamp": _float(
readback.get("enforcer_last_run_timestamp")
),
"controlled_cd_lane_live_enforcer_last_run_fresh": bool(
readback.get("enforcer_last_run_fresh") is True
),
"controlled_cd_lane_live_enforcer_apply_performed": _int(
readback.get("enforcer_apply_performed")
),
"controlled_cd_lane_live_active_job_containers": _int(
readback.get("active_job_containers")
),
"controlled_cd_lane_live_lane_process_count": _int(
readback.get("lane_process_count")
),
"controlled_cd_lane_live_root_restore_sources_left": _int(
readback.get("root_restore_sources_left")
),
"controlled_cd_lane_live_controlled_drain_active_allowed": bool(
readback.get("controlled_drain_active_allowed") is True
),
"controlled_cd_lane_live_controlled_drain_staging_allowed": bool(
readback.get("controlled_drain_staging_allowed") is True
),
"controlled_cd_lane_live_host_load5_per_core": _float(
readback.get("host_load5_per_core")
),
"controlled_cd_lane_live_gitea_actions_active_container_count": _int(
readback.get("gitea_actions_active_container_count")
),
"controlled_cd_lane_live_gitea_actions_active_process_count": _int(
readback.get("gitea_actions_active_process_count")
),
"controlled_cd_lane_live_metric_blockers": active_blockers,
}
state.update(metric_fields)
rollups = _dict(payload.setdefault("rollups", {}))
rollups.update(metric_fields)
rollups["controlled_cd_lane_live_metric_blocker_count"] = len(active_blockers)
summary = _dict(payload.setdefault("summary", {}))
summary.update(metric_fields)
summary["controlled_cd_lane_live_metric_blocker_count"] = len(active_blockers)
for item in _list(payload.get("in_progress_or_blocked_in_priority_order")):
workplan = _dict(item)
if workplan.get("workplan_id") != "P0-006":
continue
evidence = _dict(workplan.setdefault("evidence", {}))
evidence.update(metric_fields)
evidence["controlled_cd_lane_live_metric_blocker_count"] = len(
active_blockers
)
def apply_stockplatform_public_api_runtime_readback(
payload: dict[str, Any],
runtime_readback: dict[str, Any],
@@ -2116,6 +2261,87 @@ def _require_mainline_consistency(payload: dict[str, Any], label: str) -> None:
raise ValueError(f"{label}: next_execution_order must start with P0-006")
def _parse_prometheus_text_metrics(raw: str) -> dict[str, float]:
metrics: dict[str, float] = {}
allowed_names = {
"awoooi_host_gitea_actions_active_container_count",
"awoooi_host_gitea_actions_active_process_count",
"awoooi_host_load5_per_core",
"awoooi_runner_failclosed_enforcer_active_job_containers",
"awoooi_runner_failclosed_enforcer_apply_performed",
"awoooi_runner_failclosed_enforcer_controlled_drain_active_allowed",
"awoooi_runner_failclosed_enforcer_controlled_drain_staging_allowed",
"awoooi_runner_failclosed_enforcer_lane_process_count",
"awoooi_runner_failclosed_enforcer_last_run_timestamp",
"awoooi_runner_failclosed_enforcer_root_restore_sources_left",
}
for line in raw.splitlines():
if not line or line.startswith("#"):
continue
match = _METRIC_LINE_RE.match(line.strip())
if not match:
continue
name = match.group("name")
if name not in allowed_names:
continue
try:
metrics[name] = float(match.group("value"))
except ValueError:
continue
return metrics
def _build_controlled_cd_lane_metric_readback(
metrics: dict[str, float],
) -> dict[str, Any]:
last_run = metrics.get(
"awoooi_runner_failclosed_enforcer_last_run_timestamp", 0.0
)
now_seconds = datetime.now(ZoneInfo("Asia/Taipei")).timestamp()
enforcer_metric_present = (
"awoooi_runner_failclosed_enforcer_last_run_timestamp" in metrics
)
return {
"enforcer_metric_present": enforcer_metric_present,
"enforcer_last_run_timestamp": last_run,
"enforcer_last_run_age_seconds": max(0, int(now_seconds - last_run))
if last_run
else 0,
"enforcer_last_run_fresh": bool(last_run and now_seconds - last_run <= 900),
"enforcer_apply_performed": _int(
metrics.get("awoooi_runner_failclosed_enforcer_apply_performed")
),
"active_job_containers": _int(
metrics.get("awoooi_runner_failclosed_enforcer_active_job_containers")
),
"lane_process_count": _int(
metrics.get("awoooi_runner_failclosed_enforcer_lane_process_count")
),
"root_restore_sources_left": _int(
metrics.get("awoooi_runner_failclosed_enforcer_root_restore_sources_left")
),
"controlled_drain_active_allowed": _int(
metrics.get(
"awoooi_runner_failclosed_enforcer_controlled_drain_active_allowed"
)
)
== 1,
"controlled_drain_staging_allowed": _int(
metrics.get(
"awoooi_runner_failclosed_enforcer_controlled_drain_staging_allowed"
)
)
== 1,
"host_load5_per_core": _float(metrics.get("awoooi_host_load5_per_core")),
"gitea_actions_active_container_count": _int(
metrics.get("awoooi_host_gitea_actions_active_container_count")
),
"gitea_actions_active_process_count": _int(
metrics.get("awoooi_host_gitea_actions_active_process_count")
),
}
def _dict(value: Any) -> dict[str, Any]:
return value if isinstance(value, dict) else {}
@@ -2131,6 +2357,13 @@ def _int(value: Any) -> int:
return 0
def _float(value: Any) -> float:
try:
return float(str(value))
except (TypeError, ValueError):
return 0.0
def _strings(value: Any) -> list[str]:
return [str(item) for item in _list(value)]

60
apps/api/tests/test_awoooi_priority_work_order_readback_api.py
View File

@@ -14,6 +14,7 @@ from src.services.ai_agent_log_controlled_writeback_executor_readback import (
)
from src.services.awoooi_priority_work_order_readback import (
apply_ai_loop_current_blocker_execution_queue,
apply_controlled_cd_lane_live_metric_readback,
apply_harbor_registry_controlled_recovery_preflight,
apply_stockplatform_public_api_controlled_recovery_preflight,
apply_stockplatform_public_api_runtime_readback,
@@ -104,6 +105,11 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot(
"load_latest_harbor_registry_controlled_recovery_preflight",
_harbor_registry_ready,
)
monkeypatch.setattr(
agents,
"load_controlled_cd_lane_live_metric_readback",
_controlled_cd_lane_live_metrics,
)
app = FastAPI()
app.include_router(router, prefix="/api/v1")
client = TestClient(app)
@@ -124,6 +130,28 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot(
data["mainline_execution_state"]["ai_loop_current_blocker_registry_v2_ready"]
is True
)
assert (
data["mainline_execution_state"][
"controlled_cd_lane_live_enforcer_metric_present"
]
is True
)
assert (
data["mainline_execution_state"][
"controlled_cd_lane_live_enforcer_last_run_fresh"
]
is True
)
assert (
data["mainline_execution_state"][
"controlled_cd_lane_live_controlled_drain_active_allowed"
]
is False
)
assert data["summary"]["controlled_cd_lane_live_metric_status"] == (
"controlled_cd_lane_live_metric_readback_ready"
)
assert data["summary"]["controlled_cd_lane_live_metric_metadata_only"] is True
assert data["mainline_execution_state"][
"ai_loop_current_blocker_deploy_marker_readback_required"
] is False
@@ -158,6 +186,11 @@ def test_awoooi_priority_work_order_readback_endpoint_redacts_ai_loop_queue(
"load_latest_harbor_registry_controlled_recovery_preflight",
_harbor_registry_upstream_502,
)
monkeypatch.setattr(
agents,
"load_controlled_cd_lane_live_metric_readback",
_controlled_cd_lane_live_metrics,
)
app = FastAPI()
app.include_router(router, prefix="/api/v1")
client = TestClient(app)
@@ -929,6 +962,33 @@ def _stockplatform_runtime_ready() -> dict:
}
def _controlled_cd_lane_live_metrics() -> dict:
return {
"schema_version": "controlled_cd_lane_live_metric_readback_v1",
"status": "controlled_cd_lane_live_metric_readback_ready",
"source": "node_exporter_text_metrics",
"metadata_only": True,
"active_blockers": [
"controlled_cd_lane_active_not_allowed_yet",
"controlled_cd_lane_staging_not_allowed_yet",
],
"readback": {
"enforcer_metric_present": True,
"enforcer_last_run_timestamp": 1782961969.0,
"enforcer_last_run_fresh": True,
"enforcer_apply_performed": 1,
"active_job_containers": 0,
"lane_process_count": 0,
"root_restore_sources_left": 0,
"controlled_drain_active_allowed": False,
"controlled_drain_staging_allowed": False,
"host_load5_per_core": 0.64,
"gitea_actions_active_container_count": 0,
"gitea_actions_active_process_count": 0,
},
}
def _stockplatform_runtime_blocked() -> dict:
return {
"schema_version": "stockplatform_public_api_runtime_readback_v1",