wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(agents): reconcile reboot metric source blockers
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 55s
Details
CD Pipeline / build-and-deploy (push) Successful in 4m56s
Details
AI 技術雷達監控 / ai-technology-watch (push) Successful in 42s
Details
CD Pipeline / post-deploy-checks (push) Successful in 1m52s
Details

Browse Source
This commit is contained in:
Your Name
2026-07-03 01:57:11 +08:00
parent b994a87558
commit 1392369e56
2 changed files with 108 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
apps/api/src/services/reboot_auto_recovery_slo_scorecard.py
View File

@@ -55,6 +55,11 @@ _PUBLIC_MAINTENANCE_BLOCKERS = {
"public_route_raw_5xx_without_maintenance_fallback",
"public_route_unreachable_without_external_l1_fallback",
}
_PROMETHEUS_SOURCE_CONTROLLED_BLOCKERS = {
"conversation_event_hot_path_index_migration_source_missing": (
"conversation_event_hot_path_index_migration_source_present"
),
}
def load_latest_reboot_auto_recovery_slo_scorecard(
@@ -338,6 +343,8 @@ def _annotate_prometheus_metric_readback(
payload["runtime_metric_last_run_timestamp"] = _int(
metric_readback.get("last_run_timestamp")
)
payload["runtime_metric_source_control_reconciled_blockers"] = []
payload["runtime_metric_source_control_reconciled_blocker_count"] = 0
readback = _dict(payload.setdefault("readback", {}))
readback["runtime_metric_readback_present"] = present
@@ -348,6 +355,8 @@ def _annotate_prometheus_metric_readback(
readback["runtime_metric_last_run_timestamp"] = payload[
"runtime_metric_last_run_timestamp"
]
readback["runtime_metric_source_control_reconciled_blockers"] = []
readback["runtime_metric_source_control_reconciled_blocker_count"] = 0
rollups = _dict(payload.setdefault("rollups", {}))
rollups["runtime_metric_readback_present"] = present
@@ -357,6 +366,39 @@ def _annotate_prometheus_metric_readback(
rollups["runtime_metric_last_run_timestamp"] = payload[
"runtime_metric_last_run_timestamp"
]
rollups["runtime_metric_source_control_reconciled_blocker_count"] = 0
def _reconcile_prometheus_metric_active_blockers_with_source_controls(
payload: dict[str, Any],
active_blockers: list[str],
) -> list[str]:
source_controls = _dict(payload.get("source_controls"))
reconciled: list[str] = []
removed: list[str] = []
for blocker in active_blockers:
source_control_key = _PROMETHEUS_SOURCE_CONTROLLED_BLOCKERS.get(blocker)
if source_control_key and source_controls.get(source_control_key) is True:
removed.append(blocker)
continue
reconciled.append(blocker)
removed = _unique_strings(removed)
if removed:
payload["runtime_metric_source_control_reconciled_blockers"] = removed
payload["runtime_metric_source_control_reconciled_blocker_count"] = len(removed)
readback = _dict(payload.setdefault("readback", {}))
readback["runtime_metric_source_control_reconciled_blockers"] = removed
readback["runtime_metric_source_control_reconciled_blocker_count"] = len(removed)
rollups = _dict(payload.setdefault("rollups", {}))
rollups["runtime_metric_source_control_reconciled_blocker_count"] = len(
removed
)
return _unique_strings(reconciled)
def _apply_prometheus_metric_active_blockers(
@@ -368,6 +410,10 @@ def _apply_prometheus_metric_active_blockers(
active_blockers = _strings(metric_readback.get("active_blockers"))
if not active_blockers and _int(metric_readback.get("active_blocker_count")):
return
active_blockers = _reconcile_prometheus_metric_active_blockers_with_source_controls(
payload,
active_blockers,
)
can_claim_slo = metric_readback.get("ready") is True and not active_blockers
primary_blocker = str(