feat(agent): expose local console recovery phases
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 37s
CD Pipeline / build-and-deploy (push) Failing after 2m35s
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 37s
CD Pipeline / build-and-deploy (push) Failing after 2m35s
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -331,6 +331,15 @@ def _current_blocker_queue_item(recovery: dict[str, Any]) -> dict[str, Any]:
|
||||
recovery.get("controlled_recovery_package") or ""
|
||||
),
|
||||
"post_apply_verifier": str(recovery.get("post_apply_verifier") or ""),
|
||||
"controlled_local_console_execution_plan": _list_of_dicts(
|
||||
recovery.get("controlled_local_console_execution_plan")
|
||||
),
|
||||
"post_recovery_readback_commands": _list_of_strings(
|
||||
recovery.get("post_recovery_readback_commands")
|
||||
),
|
||||
"forbidden_runtime_actions": _list_of_strings(
|
||||
recovery.get("forbidden_runtime_actions")
|
||||
),
|
||||
"safe_next_step": str(recovery.get("safe_next_step") or ""),
|
||||
"runtime_write_gate": "controlled_after_110_local_console_preflight",
|
||||
"runtime_apply_required_on_110_local_console": bool(
|
||||
@@ -346,6 +355,18 @@ def _current_blocker_queue_item(recovery: dict[str, Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _list_of_dicts(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [dict(item) for item in value if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _list_of_strings(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [str(item) for item in value if str(item)]
|
||||
|
||||
|
||||
def _active_blockers(
|
||||
*,
|
||||
plan_ready: bool,
|
||||
|
||||
@@ -204,7 +204,7 @@ def _current_blocker_recovery(receipt: dict[str, Any]) -> dict[str, Any] | None:
|
||||
if not blocker_id:
|
||||
return None
|
||||
source_sample_id = str(receipt.get("source_sample_id") or "")
|
||||
return {
|
||||
recovery = {
|
||||
"source_sample_id": source_sample_id,
|
||||
"incident_id": str(classification.get("incident_id") or ""),
|
||||
"blocker_id": blocker_id,
|
||||
@@ -228,6 +228,28 @@ def _current_blocker_recovery(receipt: dict[str, Any]) -> dict[str, Any] | None:
|
||||
),
|
||||
"metadata_only": True,
|
||||
}
|
||||
recovery["controlled_local_console_execution_plan"] = _list_of_dicts(
|
||||
classification.get("controlled_local_console_execution_plan")
|
||||
)
|
||||
recovery["post_recovery_readback_commands"] = _list_of_strings(
|
||||
classification.get("post_recovery_readback_commands")
|
||||
)
|
||||
recovery["forbidden_runtime_actions"] = _list_of_strings(
|
||||
classification.get("forbidden_runtime_actions")
|
||||
)
|
||||
return recovery
|
||||
|
||||
|
||||
def _list_of_dicts(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [dict(item) for item in value if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _list_of_strings(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [str(item) for item in value if str(item)]
|
||||
|
||||
|
||||
def _target_rollups(plans: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
|
||||
@@ -53,6 +53,23 @@ def _assert_log_controlled_writeback_executor(payload: dict):
|
||||
assert current_queue["post_apply_verifier"] == (
|
||||
"check-awoooi-110-controlled-cd-lane-readiness.sh"
|
||||
)
|
||||
assert [phase["phase_id"] for phase in current_queue[
|
||||
"controlled_local_console_execution_plan"
|
||||
]] == [
|
||||
"diagnose_ssh_publickey",
|
||||
"preflight_control_path_and_harbor",
|
||||
"repair_ssh_metadata_if_check_confirms_metadata_drift",
|
||||
"repair_harbor_once_if_v2_still_502",
|
||||
"verify_controlled_cd_lane",
|
||||
]
|
||||
assert current_queue["post_recovery_readback_commands"] == [
|
||||
"read-public-gitea-actions-queue.py --json",
|
||||
"curl -k https://registry.wooo.work/v2/",
|
||||
"curl http://192.168.0.110:5000/v2/",
|
||||
]
|
||||
assert "workflow_dispatch_from_this_endpoint" in current_queue[
|
||||
"forbidden_runtime_actions"
|
||||
]
|
||||
assert current_queue["external_control_path_blocker"] == (
|
||||
"publickey_offer_timeout"
|
||||
)
|
||||
|
||||
@@ -23,10 +23,11 @@ def test_log_controlled_writeback_executor_endpoint_returns_readback():
|
||||
response = client.get("/api/v1/agents/agent-log-controlled-writeback-executor-readback")
|
||||
|
||||
assert response.status_code == 200
|
||||
_assert_executor_readback(response.json())
|
||||
assert "192.168.0.110" not in response.text
|
||||
_assert_executor_readback(response.json(), public_endpoint=True)
|
||||
|
||||
|
||||
def _assert_executor_readback(payload: dict):
|
||||
def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
expected_runtime_sample_count = 3
|
||||
expected_target_count = 6
|
||||
expected_writeback_plan_count = expected_runtime_sample_count * expected_target_count
|
||||
@@ -119,6 +120,29 @@ def _assert_executor_readback(payload: dict):
|
||||
assert current_queue[0]["post_apply_verifier"] == (
|
||||
"check-awoooi-110-controlled-cd-lane-readiness.sh"
|
||||
)
|
||||
assert [phase["phase_id"] for phase in current_queue[0][
|
||||
"controlled_local_console_execution_plan"
|
||||
]] == [
|
||||
"diagnose_ssh_publickey",
|
||||
"preflight_control_path_and_harbor",
|
||||
"repair_ssh_metadata_if_check_confirms_metadata_drift",
|
||||
"repair_harbor_once_if_v2_still_502",
|
||||
"verify_controlled_cd_lane",
|
||||
]
|
||||
assert current_queue[0]["controlled_local_console_execution_plan"][0][
|
||||
"mode"
|
||||
] == "read_only"
|
||||
assert current_queue[0]["controlled_local_console_execution_plan"][3][
|
||||
"command"
|
||||
] == "recover-110-control-path-and-harbor-local.sh --repair-harbor-once"
|
||||
assert current_queue[0]["post_recovery_readback_commands"] == [
|
||||
"read-public-gitea-actions-queue.py --json",
|
||||
"curl -k https://registry.wooo.work/v2/",
|
||||
_expected_local_registry_readback(public_endpoint=public_endpoint),
|
||||
]
|
||||
assert "read_runner_registration_token_or_runner_file" in current_queue[0][
|
||||
"forbidden_runtime_actions"
|
||||
]
|
||||
assert current_queue[0]["runtime_write_gate"] == (
|
||||
"controlled_after_110_local_console_preflight"
|
||||
)
|
||||
@@ -154,3 +178,9 @@ def _assert_executor_readback(payload: dict):
|
||||
assert boundaries["raw_log_payload_persisted"] is False
|
||||
assert boundaries["secret_value_collection_allowed"] is False
|
||||
assert boundaries["github_api_used"] is False
|
||||
|
||||
|
||||
def _expected_local_registry_readback(*, public_endpoint: bool = False) -> str:
|
||||
if public_endpoint:
|
||||
return "curl host:public-gateway/registry/v2/"
|
||||
return "curl http://192.168.0.110:5000/v2/"
|
||||
|
||||
@@ -23,10 +23,11 @@ def test_log_controlled_writeback_plan_endpoint_returns_readback():
|
||||
response = client.get("/api/v1/agents/agent-log-controlled-writeback-plan-readback")
|
||||
|
||||
assert response.status_code == 200
|
||||
_assert_controlled_writeback_plan(response.json())
|
||||
assert "192.168.0.110" not in response.text
|
||||
_assert_controlled_writeback_plan(response.json(), public_endpoint=True)
|
||||
|
||||
|
||||
def _assert_controlled_writeback_plan(payload: dict):
|
||||
def _assert_controlled_writeback_plan(payload: dict, *, public_endpoint: bool = False):
|
||||
assert payload["schema_version"] == "ai_agent_log_controlled_writeback_plan_readback_v1"
|
||||
assert payload["priority"] == "P1-LOG-KM-RAG-MCP-PLAYBOOK"
|
||||
assert payload["status"] == "controlled_writeback_plan_ready"
|
||||
@@ -132,6 +133,32 @@ def _assert_controlled_writeback_plan(payload: dict):
|
||||
plan["current_blocker_recovery"]["post_apply_verifier"]
|
||||
for plan in p0_plans
|
||||
} == {"check-awoooi-110-controlled-cd-lane-readiness.sh"}
|
||||
assert all(
|
||||
[
|
||||
phase["phase_id"]
|
||||
for phase in plan["current_blocker_recovery"][
|
||||
"controlled_local_console_execution_plan"
|
||||
]
|
||||
]
|
||||
== [
|
||||
"diagnose_ssh_publickey",
|
||||
"preflight_control_path_and_harbor",
|
||||
"repair_ssh_metadata_if_check_confirms_metadata_drift",
|
||||
"repair_harbor_once_if_v2_still_502",
|
||||
"verify_controlled_cd_lane",
|
||||
]
|
||||
for plan in p0_plans
|
||||
)
|
||||
assert all(
|
||||
plan["current_blocker_recovery"]["post_recovery_readback_commands"]
|
||||
== _expected_post_recovery_readback_commands(public_endpoint=public_endpoint)
|
||||
for plan in p0_plans
|
||||
)
|
||||
assert all(
|
||||
"read_or_print_authorized_keys"
|
||||
in plan["current_blocker_recovery"]["forbidden_runtime_actions"]
|
||||
for plan in p0_plans
|
||||
)
|
||||
assert all(
|
||||
plan["current_blocker_recovery"]["runtime_apply_required_on_110_local_console"]
|
||||
is True
|
||||
@@ -153,3 +180,18 @@ def _assert_controlled_writeback_plan(payload: dict):
|
||||
assert boundaries["raw_log_payload_persisted"] is False
|
||||
assert boundaries["secret_value_collection_allowed"] is False
|
||||
assert boundaries["github_api_used"] is False
|
||||
|
||||
|
||||
def _expected_post_recovery_readback_commands(
|
||||
*, public_endpoint: bool = False
|
||||
) -> list[str]:
|
||||
local_registry_readback = (
|
||||
"curl host:public-gateway/registry/v2/"
|
||||
if public_endpoint
|
||||
else "curl http://192.168.0.110:5000/v2/"
|
||||
)
|
||||
return [
|
||||
"read-public-gitea-actions-queue.py --json",
|
||||
"curl -k https://registry.wooo.work/v2/",
|
||||
local_registry_readback,
|
||||
]
|
||||
|
||||
@@ -101,6 +101,26 @@ def _assert_feedback_dry_run_payload(payload: dict):
|
||||
assert {
|
||||
receipt["classification"]["post_apply_verifier"] for receipt in p0_receipts
|
||||
} == {"check-awoooi-110-controlled-cd-lane-readiness.sh"}
|
||||
assert all(
|
||||
len(receipt["classification"]["controlled_local_console_execution_plan"]) == 5
|
||||
for receipt in p0_receipts
|
||||
)
|
||||
assert all(
|
||||
receipt["classification"]["controlled_local_console_execution_plan"][3][
|
||||
"command"
|
||||
]
|
||||
== "recover-110-control-path-and-harbor-local.sh --repair-harbor-once"
|
||||
for receipt in p0_receipts
|
||||
)
|
||||
assert all(
|
||||
len(receipt["classification"]["post_recovery_readback_commands"]) == 3
|
||||
for receipt in p0_receipts
|
||||
)
|
||||
assert all(
|
||||
"restart_docker_daemon"
|
||||
in receipt["classification"]["forbidden_runtime_actions"]
|
||||
for receipt in p0_receipts
|
||||
)
|
||||
|
||||
boundaries = payload["operation_boundaries"]
|
||||
assert boundaries["dry_run_only"] is True
|
||||
|
||||
@@ -115,6 +115,25 @@ def _assert_log_intelligence_payload(payload: dict):
|
||||
assert p0_sample["classification"]["post_apply_verifier"] == (
|
||||
"check-awoooi-110-controlled-cd-lane-readiness.sh"
|
||||
)
|
||||
recovery_plan = p0_sample["classification"][
|
||||
"controlled_local_console_execution_plan"
|
||||
]
|
||||
assert [phase["phase_id"] for phase in recovery_plan] == [
|
||||
"diagnose_ssh_publickey",
|
||||
"preflight_control_path_and_harbor",
|
||||
"repair_ssh_metadata_if_check_confirms_metadata_drift",
|
||||
"repair_harbor_once_if_v2_still_502",
|
||||
"verify_controlled_cd_lane",
|
||||
]
|
||||
assert recovery_plan[0]["mode"] == "read_only"
|
||||
assert recovery_plan[2]["mode"] == "controlled_apply"
|
||||
assert recovery_plan[2]["runtime_write_allowed_only_on_110_local_console"] is True
|
||||
assert "read-public-gitea-actions-queue.py --json" in p0_sample[
|
||||
"classification"
|
||||
]["post_recovery_readback_commands"]
|
||||
assert "read_runner_registration_token_or_runner_file" in p0_sample[
|
||||
"classification"
|
||||
]["forbidden_runtime_actions"]
|
||||
assert p0_sample["raw_log_payload_persisted"] is False
|
||||
|
||||
label_groups = {
|
||||
|
||||
@@ -109,6 +109,56 @@
|
||||
"registry_v2_status": 502,
|
||||
"controlled_recovery_package": "recover-110-control-path-and-harbor-local.sh --check",
|
||||
"post_apply_verifier": "check-awoooi-110-controlled-cd-lane-readiness.sh",
|
||||
"controlled_local_console_execution_plan": [
|
||||
{
|
||||
"phase_id": "diagnose_ssh_publickey",
|
||||
"mode": "read_only",
|
||||
"command": "diagnose-110-ssh-publickey-auth.sh",
|
||||
"expected_result": "ssh_publickey_classifier_readback_without_key_material",
|
||||
"runtime_write_allowed_only_on_110_local_console": false
|
||||
},
|
||||
{
|
||||
"phase_id": "preflight_control_path_and_harbor",
|
||||
"mode": "check",
|
||||
"command": "recover-110-control-path-and-harbor-local.sh --check",
|
||||
"expected_result": "ssh_metadata_harbor_watchdog_and_cd_lane_readiness_readback",
|
||||
"runtime_write_allowed_only_on_110_local_console": false
|
||||
},
|
||||
{
|
||||
"phase_id": "repair_ssh_metadata_if_check_confirms_metadata_drift",
|
||||
"mode": "controlled_apply",
|
||||
"command": "recover-110-control-path-and-harbor-local.sh --apply-ssh-metadata",
|
||||
"expected_result": "authorized_keys_metadata_repaired_without_key_material_read",
|
||||
"runtime_write_allowed_only_on_110_local_console": true
|
||||
},
|
||||
{
|
||||
"phase_id": "repair_harbor_once_if_v2_still_502",
|
||||
"mode": "controlled_apply",
|
||||
"command": "recover-110-control-path-and-harbor-local.sh --repair-harbor-once",
|
||||
"expected_result": "harbor_local_v2_returns_200_or_401_after_single_bounded_repair",
|
||||
"runtime_write_allowed_only_on_110_local_console": true
|
||||
},
|
||||
{
|
||||
"phase_id": "verify_controlled_cd_lane",
|
||||
"mode": "post_apply_verifier",
|
||||
"command": "check-awoooi-110-controlled-cd-lane-readiness.sh",
|
||||
"expected_result": "awoooi_host_runner_ready_without_generic_labels_or_secret_reads",
|
||||
"runtime_write_allowed_only_on_110_local_console": false
|
||||
}
|
||||
],
|
||||
"post_recovery_readback_commands": [
|
||||
"read-public-gitea-actions-queue.py --json",
|
||||
"curl -k https://registry.wooo.work/v2/",
|
||||
"curl http://192.168.0.110:5000/v2/"
|
||||
],
|
||||
"forbidden_runtime_actions": [
|
||||
"read_or_print_authorized_keys",
|
||||
"read_runner_registration_token_or_runner_file",
|
||||
"restart_docker_daemon",
|
||||
"reboot_host",
|
||||
"docker_system_prune",
|
||||
"workflow_dispatch_from_this_endpoint"
|
||||
],
|
||||
"safe_next_step": "run_110_local_recovery_package_then_rerun_harbor_queue_and_registry_readback"
|
||||
},
|
||||
"raw_log_payload_persisted": false
|
||||
|
||||
Reference in New Issue
Block a user