fix(agent): queue bounded host pressure handoff
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m28s
CD Pipeline / build-and-deploy (push) Successful in 6m49s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 13s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m28s
CD Pipeline / build-and-deploy (push) Successful in 6m49s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 13s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s
This commit is contained in:
@@ -1225,9 +1225,21 @@ class RepairCandidateService:
|
|||||||
}
|
}
|
||||||
and repair_template.startswith("agent99-mode ")
|
and repair_template.startswith("agent99-mode ")
|
||||||
)
|
)
|
||||||
|
rollback_template = str(
|
||||||
|
template.get("rollback_command_template") or ""
|
||||||
|
).strip()
|
||||||
|
safe_host_pressure_readonly_route = bool(
|
||||||
|
route == "host_pressure_readonly_after_owner_review"
|
||||||
|
and repair_template.startswith(
|
||||||
|
"/home/wooo/scripts/host-sustained-load-controller.py "
|
||||||
|
)
|
||||||
|
and "systemctl restart" not in repair_template.lower()
|
||||||
|
and rollback_template.startswith("no host mutation")
|
||||||
|
)
|
||||||
if (
|
if (
|
||||||
"playbook_command_not_safely_routable" in set(blockers)
|
"playbook_command_not_safely_routable" in set(blockers)
|
||||||
and not safe_agent99_route
|
and not safe_agent99_route
|
||||||
|
and not safe_host_pressure_readonly_route
|
||||||
):
|
):
|
||||||
return False
|
return False
|
||||||
if not coverage_gap.get("mcp_evidence_ready"):
|
if not coverage_gap.get("mcp_evidence_ready"):
|
||||||
@@ -1241,7 +1253,9 @@ class RepairCandidateService:
|
|||||||
return False
|
return False
|
||||||
if repair_template.startswith("service-specific repair"):
|
if repair_template.startswith("service-specific repair"):
|
||||||
return False
|
return False
|
||||||
return route.endswith("_after_owner_review")
|
return safe_host_pressure_readonly_route or route.endswith(
|
||||||
|
"_after_owner_review"
|
||||||
|
)
|
||||||
|
|
||||||
def _build_playbook_draft_template(
|
def _build_playbook_draft_template(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -559,6 +559,55 @@ def test_host_pressure_controller_is_never_drafted_as_systemd_restart() -> None:
|
|||||||
assert "systemctl is-active" not in " ".join(template["verifier_plan_template"])
|
assert "systemctl is-active" not in " ".join(template["verifier_plan_template"])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_host_pressure_readonly_draft_queues_dedicated_ansible_candidate() -> None:
|
||||||
|
incident = _incident()
|
||||||
|
incident.signals[0].alert_name = "Host110SustainedModeratePressureLivefire"
|
||||||
|
incident.signals[0].labels = {
|
||||||
|
"component": "host-pressure-controller-livefire-source-sha",
|
||||||
|
"host": "110",
|
||||||
|
}
|
||||||
|
incident.affected_services = ["host-pressure-controller-livefire-source-sha"]
|
||||||
|
recorded: list[dict] = []
|
||||||
|
|
||||||
|
async def fake_recorder(**kwargs):
|
||||||
|
recorded.append(kwargs)
|
||||||
|
return True
|
||||||
|
|
||||||
|
service = RepairCandidateService(
|
||||||
|
incident_service=FakeIncidentService(),
|
||||||
|
investigator=FakeInvestigator(_evidence(incident.incident_id)),
|
||||||
|
playbook_repository=FakePlaybookRepository(_generic_fallback_playbook()),
|
||||||
|
auto_repair_service=FakeAutoRepairService(),
|
||||||
|
ansible_audit_recorder=fake_recorder,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await service.build_from_incident(
|
||||||
|
incident=incident,
|
||||||
|
alertname="Host110SustainedModeratePressureLivefire",
|
||||||
|
target_resource="host-pressure-controller-livefire-source-sha",
|
||||||
|
namespace="",
|
||||||
|
message="controlled host pressure live-fire",
|
||||||
|
fallback_action="NO_ACTION - REPAIR_CANDIDATE_MISSING",
|
||||||
|
matched_playbook_id="PB-GENERIC-FALLBACK",
|
||||||
|
severity="medium",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.draft_ready_for_owner_review is True
|
||||||
|
assert result.metadata["repair_candidate_owner_review_required"] is False
|
||||||
|
handoff = result.metadata["controlled_executor_handoff"]
|
||||||
|
assert handoff["status"] == "ansible_candidate_matched_queued"
|
||||||
|
assert handoff["check_mode_queued"] is True
|
||||||
|
assert handoff["candidate_count"] >= 1
|
||||||
|
assert recorded
|
||||||
|
|
||||||
|
audit_input = recorded[0]["proposal_data"]
|
||||||
|
assert audit_input["route_id"] == "host_pressure_readonly_after_owner_review"
|
||||||
|
assert audit_input["action"].startswith(
|
||||||
|
"/home/wooo/scripts/host-sustained-load-controller.py"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_postgres_slow_query_gap_prefills_database_owner_review_not_restart() -> None:
|
async def test_postgres_slow_query_gap_prefills_database_owner_review_not_restart() -> None:
|
||||||
incident = _incident()
|
incident = _incident()
|
||||||
|
|||||||
Reference in New Issue
Block a user