fix(recovery): propagate 110 high load blocker
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 39s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 39s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
This commit is contained in:
@@ -103,6 +103,11 @@ def load_latest_ai_agent_log_controlled_writeback_executor_readback() -> dict[st
|
||||
for item in current_blocker_queue
|
||||
if item["external_control_path_blocker"]
|
||||
),
|
||||
"current_blocker_control_path_pressure_blocked_count": sum(
|
||||
1
|
||||
for item in current_blocker_queue
|
||||
if item["control_path_pressure_blocker"]
|
||||
),
|
||||
"current_blocker_local_recovery_package_count": sum(
|
||||
1 for item in current_blocker_queue if item["controlled_recovery_package"]
|
||||
),
|
||||
@@ -322,6 +327,10 @@ def _current_blocker_queue_item(recovery: dict[str, Any]) -> dict[str, Any]:
|
||||
blocker_id = str(recovery.get("blocker_id") or "")
|
||||
source_sample_id = str(recovery.get("source_sample_id") or "")
|
||||
external_blocker = str(recovery.get("external_control_path_blocker") or "")
|
||||
node_load_classifier = str(recovery.get("node_load_classifier") or "")
|
||||
control_path_pressure_blocker = (
|
||||
"node_load_high" if node_load_classifier == "high_load" else ""
|
||||
)
|
||||
return {
|
||||
"queue_item_id": f"current-p0-blocker::{blocker_id}",
|
||||
"source_sample_id": source_sample_id,
|
||||
@@ -330,6 +339,8 @@ def _current_blocker_queue_item(recovery: dict[str, Any]) -> dict[str, Any]:
|
||||
"status": "controlled_recovery_packaged_waiting_control_path_readback",
|
||||
"risk_tier": str(recovery.get("risk_tier") or "unknown"),
|
||||
"runner_label": str(recovery.get("runner_label") or ""),
|
||||
"node_load_classifier": node_load_classifier,
|
||||
"node_load_high": node_load_classifier == "high_load",
|
||||
"registry_v2_status": recovery.get("registry_v2_status"),
|
||||
"controlled_recovery_package": str(
|
||||
recovery.get("controlled_recovery_package") or ""
|
||||
@@ -357,6 +368,7 @@ def _current_blocker_queue_item(recovery: dict[str, Any]) -> dict[str, Any]:
|
||||
recovery.get("runtime_apply_required_on_110_local_console") is True
|
||||
),
|
||||
"external_control_path_blocker": external_blocker,
|
||||
"control_path_pressure_blocker": control_path_pressure_blocker,
|
||||
"control_channel_readback_required": bool(external_blocker),
|
||||
"metadata_only_writeback_ready": True,
|
||||
"runtime_dispatch_performed": False,
|
||||
|
||||
@@ -544,6 +544,8 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
]
|
||||
forbidden_runtime_actions = _strings(first_item.get("forbidden_runtime_actions"))
|
||||
external_blocker = str(first_item.get("external_control_path_blocker") or "")
|
||||
pressure_blocker = str(first_item.get("control_path_pressure_blocker") or "")
|
||||
node_load_classifier = str(first_item.get("node_load_classifier") or "")
|
||||
runtime_write_gate = str(first_item.get("runtime_write_gate") or "")
|
||||
|
||||
state = _dict(payload.setdefault("mainline_execution_state", {}))
|
||||
@@ -557,6 +559,10 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
state["ai_loop_current_blocker_execution_queue_count"] = len(queue)
|
||||
state["ai_loop_current_blocker_id"] = blocker_id
|
||||
state["ai_loop_current_blocker_control_path_blocker"] = external_blocker
|
||||
state["ai_loop_current_blocker_control_path_pressure_blocker"] = (
|
||||
pressure_blocker
|
||||
)
|
||||
state["ai_loop_current_blocker_node_load_classifier"] = node_load_classifier
|
||||
state["ai_loop_current_blocker_runtime_write_gate"] = runtime_write_gate
|
||||
state["ai_loop_current_blocker_learning_target_count"] = len(learning_targets)
|
||||
state["ai_loop_current_blocker_local_console_phase_count"] = len(
|
||||
@@ -582,6 +588,7 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
+ ["ai_loop_current_blocker_execution_queue"]
|
||||
+ ([blocker_id] if blocker_id else [])
|
||||
+ ([external_blocker] if external_blocker else [])
|
||||
+ ([pressure_blocker] if pressure_blocker else [])
|
||||
)
|
||||
|
||||
for item in _list(payload.get("in_progress_or_blocked_in_priority_order")):
|
||||
@@ -609,6 +616,12 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
)
|
||||
evidence["ai_loop_current_blocker_runtime_write_gate"] = runtime_write_gate
|
||||
evidence["ai_loop_current_blocker_control_path_blocker"] = external_blocker
|
||||
evidence["ai_loop_current_blocker_control_path_pressure_blocker"] = (
|
||||
pressure_blocker
|
||||
)
|
||||
evidence["ai_loop_current_blocker_node_load_classifier"] = (
|
||||
node_load_classifier
|
||||
)
|
||||
evidence["ai_loop_current_blocker_learning_writeback_targets"] = (
|
||||
learning_targets
|
||||
)
|
||||
@@ -698,6 +711,10 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
summary["ai_loop_current_blocker_execution_queue_count"] = len(queue)
|
||||
summary["ai_loop_current_blocker_id"] = blocker_id
|
||||
summary["ai_loop_current_blocker_control_path_blocker"] = external_blocker
|
||||
summary["ai_loop_current_blocker_control_path_pressure_blocker"] = (
|
||||
pressure_blocker
|
||||
)
|
||||
summary["ai_loop_current_blocker_node_load_classifier"] = node_load_classifier
|
||||
summary["ai_loop_current_blocker_controlled_recovery_package"] = (
|
||||
controlled_recovery_package
|
||||
)
|
||||
|
||||
@@ -267,9 +267,9 @@ def _local_console_phase_readback(
|
||||
),
|
||||
_phase(
|
||||
"preflight_control_path_and_harbor",
|
||||
_phase_status(
|
||||
ready=watchdog_check["receipt_seen"],
|
||||
blocked_status="blocked_waiting_harbor_watchdog_check_receipt",
|
||||
_preflight_control_path_phase_status(
|
||||
ssh_diagnosis=ssh_diagnosis,
|
||||
watchdog_check=watchdog_check,
|
||||
),
|
||||
"watchdog_check",
|
||||
),
|
||||
@@ -328,6 +328,18 @@ def _phase_status(*, ready: bool, blocked_status: str) -> str:
|
||||
return "ready" if ready else blocked_status
|
||||
|
||||
|
||||
def _preflight_control_path_phase_status(
|
||||
*,
|
||||
ssh_diagnosis: dict[str, Any],
|
||||
watchdog_check: dict[str, Any],
|
||||
) -> str:
|
||||
if watchdog_check["receipt_seen"]:
|
||||
return "ready"
|
||||
if ssh_diagnosis["receipt_seen"] and ssh_diagnosis["node_high_load_seen"]:
|
||||
return "blocked_waiting_110_load_to_normalize_before_harbor_preflight"
|
||||
return "blocked_waiting_harbor_watchdog_check_receipt"
|
||||
|
||||
|
||||
def _ssh_metadata_phase_status(
|
||||
*,
|
||||
ssh_diagnosis: dict[str, Any],
|
||||
@@ -394,14 +406,15 @@ def _parse_ssh_publickey_diagnosis_output(output: str) -> dict[str, Any]:
|
||||
and item["classifier"] == "systemctl_show_timeout"
|
||||
for item in systemd_units
|
||||
)
|
||||
node_load_classifier = str(fields.get("NODE_LOAD_CLASSIFIER") or "")
|
||||
node_high_load_seen = node_load_classifier == "high_load"
|
||||
ssh_port_tcp_open = str(fields.get("SSH_PORT") or "") == "tcp_open"
|
||||
node_exporter_ok = str(fields.get("NODE_EXPORTER") or "") == "ok"
|
||||
node_high_load_seen = str(fields.get("NODE_LOAD_CLASSIFIER") or "") == "high_load"
|
||||
return {
|
||||
"receipt_seen": marker_seen,
|
||||
"target": str(fields.get("TARGET") or ""),
|
||||
"node_exporter_ok": node_exporter_ok,
|
||||
"node_load_classifier": str(fields.get("NODE_LOAD_CLASSIFIER") or ""),
|
||||
"node_load_classifier": node_load_classifier,
|
||||
"node_load1_per_cpu": _float_or_none(fields.get("NODE_LOAD1_PER_CPU")),
|
||||
"node_high_load_seen": node_high_load_seen,
|
||||
"ssh_port_tcp_open": ssh_port_tcp_open,
|
||||
|
||||
@@ -35,6 +35,10 @@ def _assert_log_controlled_writeback_executor(payload: dict):
|
||||
assert payload["rollups"]["controlled_apply_enabled_by_policy"] is True
|
||||
assert payload["rollups"]["current_blocker_execution_queue_count"] == 1
|
||||
assert payload["rollups"]["current_blocker_control_path_blocked_count"] == 1
|
||||
assert (
|
||||
payload["rollups"]["current_blocker_control_path_pressure_blocked_count"]
|
||||
== 1
|
||||
)
|
||||
assert payload["rollups"]["current_blocker_local_recovery_package_count"] == 1
|
||||
assert payload["rollups"]["runtime_dispatch_performed"] is False
|
||||
|
||||
@@ -73,6 +77,9 @@ def _assert_log_controlled_writeback_executor(payload: dict):
|
||||
assert current_queue["external_control_path_blocker"] == (
|
||||
"publickey_offer_timeout"
|
||||
)
|
||||
assert current_queue["node_load_classifier"] == "high_load"
|
||||
assert current_queue["node_load_high"] is True
|
||||
assert current_queue["control_path_pressure_blocker"] == "node_load_high"
|
||||
assert context["raw_payload_required"] is False
|
||||
|
||||
boundaries = payload["operation_boundaries"]
|
||||
|
||||
@@ -65,6 +65,10 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
assert payload["rollups"]["controlled_apply_enabled_by_policy"] is True
|
||||
assert payload["rollups"]["current_blocker_execution_queue_count"] == 1
|
||||
assert payload["rollups"]["current_blocker_control_path_blocked_count"] == 1
|
||||
assert (
|
||||
payload["rollups"]["current_blocker_control_path_pressure_blocked_count"]
|
||||
== 1
|
||||
)
|
||||
assert payload["rollups"]["current_blocker_local_recovery_package_count"] == 1
|
||||
assert (
|
||||
payload["rollups"]["current_blocker_harbor_recovery_receipt_input_count"]
|
||||
@@ -117,6 +121,8 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
"controlled_recovery_packaged_waiting_control_path_readback"
|
||||
)
|
||||
assert current_queue[0]["runner_label"] == "awoooi-host"
|
||||
assert current_queue[0]["node_load_classifier"] == "high_load"
|
||||
assert current_queue[0]["node_load_high"] is True
|
||||
assert current_queue[0]["registry_v2_status"] == 502
|
||||
assert current_queue[0]["controlled_recovery_package"] == (
|
||||
"recover-110-control-path-and-harbor-local.sh --check"
|
||||
@@ -180,6 +186,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
assert current_queue[0]["external_control_path_blocker"] == (
|
||||
"publickey_offer_timeout"
|
||||
)
|
||||
assert current_queue[0]["control_path_pressure_blocker"] == "node_load_high"
|
||||
assert current_queue[0]["control_channel_readback_required"] is True
|
||||
assert set(current_queue[0]["learning_writeback_targets"]) == set(batches)
|
||||
assert len(current_queue[0]["target_batches"]) == 6
|
||||
|
||||
@@ -263,6 +263,10 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
assert state["ai_loop_current_blocker_control_path_blocker"] == (
|
||||
"publickey_offer_timeout"
|
||||
)
|
||||
assert state["ai_loop_current_blocker_control_path_pressure_blocker"] == (
|
||||
"node_load_high"
|
||||
)
|
||||
assert state["ai_loop_current_blocker_node_load_classifier"] == "high_load"
|
||||
assert evidence["ai_loop_current_blocker_execution_queue_count"] == 1
|
||||
assert evidence["ai_loop_current_blocker_id"] == (
|
||||
"harbor_110_repair_no_matching_runner"
|
||||
@@ -281,6 +285,10 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
assert evidence["ai_loop_current_blocker_control_path_blocker"] == (
|
||||
"publickey_offer_timeout"
|
||||
)
|
||||
assert evidence["ai_loop_current_blocker_control_path_pressure_blocker"] == (
|
||||
"node_load_high"
|
||||
)
|
||||
assert evidence["ai_loop_current_blocker_node_load_classifier"] == "high_load"
|
||||
assert set(evidence["ai_loop_current_blocker_learning_writeback_targets"]) == {
|
||||
"km",
|
||||
"rag",
|
||||
@@ -350,6 +358,12 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
assert payload["summary"]["ai_loop_current_blocker_id"] == (
|
||||
"harbor_110_repair_no_matching_runner"
|
||||
)
|
||||
assert payload["summary"]["ai_loop_current_blocker_control_path_pressure_blocker"] == (
|
||||
"node_load_high"
|
||||
)
|
||||
assert payload["summary"]["ai_loop_current_blocker_node_load_classifier"] == (
|
||||
"high_load"
|
||||
)
|
||||
assert payload["summary"]["ai_loop_current_blocker_local_console_phase_count"] == 5
|
||||
assert payload["summary"][
|
||||
"ai_loop_current_blocker_post_recovery_readback_command_count"
|
||||
|
||||
@@ -198,6 +198,33 @@ def test_harbor_recovery_receipt_classifies_publickey_offer_timeout() -> None:
|
||||
] == "blocked_waiting_ssh_metadata_repair_receipt_after_publickey_timeout"
|
||||
|
||||
|
||||
def test_harbor_recovery_receipt_classifies_110_high_load() -> None:
|
||||
diagnosis_output = _ssh_publickey_diagnosis_output().replace(
|
||||
"NODE_LOAD_CLASSIFIER=load_not_high",
|
||||
"NODE_LOAD_CLASSIFIER=high_load",
|
||||
)
|
||||
|
||||
payload = validate_harbor_registry_controlled_recovery_receipt(
|
||||
{
|
||||
"ssh_publickey_diagnosis_output": diagnosis_output,
|
||||
}
|
||||
)
|
||||
|
||||
assert "ssh_publickey_node_high_load_on_110" in payload["active_blockers"]
|
||||
diagnosis = payload["readback"]["ssh_publickey_diagnosis"]
|
||||
assert diagnosis["node_load_classifier"] == "high_load"
|
||||
assert diagnosis["node_high_load_seen"] is True
|
||||
phases = {
|
||||
phase["phase_id"]: phase
|
||||
for phase in payload["local_console_phase_readback"]["phases"]
|
||||
}
|
||||
assert phases["preflight_control_path_and_harbor"]["status"] == (
|
||||
"blocked_waiting_110_load_to_normalize_before_harbor_preflight"
|
||||
)
|
||||
assert payload["rollups"]["ssh_publickey_node_load_classifier"] == "high_load"
|
||||
assert payload["rollups"]["ssh_publickey_node_high_load_seen"] is True
|
||||
|
||||
|
||||
def test_harbor_recovery_receipt_surfaces_gitea_queue_blockers() -> None:
|
||||
payload = validate_harbor_registry_controlled_recovery_receipt(
|
||||
{
|
||||
|
||||
@@ -84,6 +84,7 @@
|
||||
"harbor_registry_public_route_unavailable",
|
||||
"harbor_110_repair_no_matching_runner",
|
||||
"ssh_publickey_offer_timeout",
|
||||
"node_load_high",
|
||||
"controlled_lane_verifier_packaged",
|
||||
"local_recovery_package_packaged"
|
||||
],
|
||||
@@ -105,7 +106,7 @@
|
||||
"runner_label": "awoooi-host",
|
||||
"current_blocker": "harbor_110_repair_no_matching_runner",
|
||||
"ssh_auth_classification": "publickey_offer_timeout",
|
||||
"node_load_classifier": "load_not_high",
|
||||
"node_load_classifier": "high_load",
|
||||
"registry_v2_status": 502,
|
||||
"controlled_recovery_package": "recover-110-control-path-and-harbor-local.sh --check",
|
||||
"post_apply_verifier": "check-awoooi-110-controlled-cd-lane-readiness.sh",
|
||||
|
||||
Reference in New Issue
Block a user