fix(agent): route disk pressure to bounded repair
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled

This commit is contained in:
ogt
2026-07-15 00:42:23 +08:00
parent dc90bcc44e
commit 8862da3929
9 changed files with 1145 additions and 31 deletions

View File

@@ -417,6 +417,11 @@ _CATALOG: tuple[dict[str, Any], ...] = (
"gitea_service",
"systemd_control_plane",
],
"activation_keywords": [
"host110sustainedmoderatepressure",
"host-pressure-controller",
"sustained pressure",
],
"supports_check_mode": True,
"auto_apply_enabled": False,
"approval_required": False,
@@ -424,6 +429,74 @@ _CATALOG: tuple[dict[str, Any], ...] = (
"no_write_only": True,
"catalog_revision": "2026-07-14-host110-inputs-v2",
},
{
"catalog_id": "ansible:110-disk-pressure",
"playbook_path": (
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
),
"check_mode_playbook_path": (
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
),
"inventory_hosts": ["host_110"],
"domains": ["disk_pressure", "docker_artifacts", "host_110"],
"keywords": [
"hostoutofdiskspace",
"hostdiskusagehigh",
"hostdiskusagecritical",
"hostdiskwillfillin24hours",
"disk_full",
"node-exporter-110",
"192.168.0.110",
],
"activation_keyword_groups": [
[
"hostoutofdiskspace",
"hostdiskusagehigh",
"hostdiskusagecritical",
"hostdiskwillfillin24hours",
],
["node-exporter-110", "192.168.0.110", "host_110", "host110"],
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
"catalog_revision": "2026-07-14-disk-pressure-bounded-v1",
},
{
"catalog_id": "ansible:188-disk-pressure",
"playbook_path": (
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
),
"check_mode_playbook_path": (
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
),
"inventory_hosts": ["host_188"],
"domains": ["disk_pressure", "docker_artifacts", "host_188"],
"keywords": [
"hostoutofdiskspace",
"hostdiskusagehigh",
"hostdiskusagecritical",
"hostdiskwillfillin24hours",
"disk_full",
"node-exporter-188",
"192.168.0.188",
],
"activation_keyword_groups": [
[
"hostoutofdiskspace",
"hostdiskusagehigh",
"hostdiskusagecritical",
"hostdiskwillfillin24hours",
],
["node-exporter-188", "192.168.0.188", "host_188", "host188"],
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
"catalog_revision": "2026-07-14-disk-pressure-bounded-v1",
},
{
"catalog_id": "ansible:110-devops",
"playbook_path": "infra/ansible/playbooks/110-node-exporter-repair.yml",
@@ -437,9 +510,13 @@ _CATALOG: tuple[dict[str, Any], ...] = (
"node_exporter",
"node-exporter",
],
"activation_keywords": [
"node-exporter-110",
"node exporter 110",
"activation_keyword_groups": [
[
"nodeexporterdown",
"nodeexporterscrapedown",
"node_exporter_down",
],
["node-exporter-110", "192.168.0.110", "host_110", "host110"],
],
"supports_check_mode": True,
"auto_apply_enabled": True,
@@ -914,7 +991,15 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
matched_activation_keywords = [
keyword for keyword in activation_keywords if keyword in haystack
]
semantic_match = not activation_keywords or bool(matched_activation_keywords)
activation_keyword_groups = item.get("activation_keyword_groups") or []
matched_activation_keyword_groups = [
[keyword for keyword in group if keyword in haystack]
for group in activation_keyword_groups
]
semantic_match = (
(not activation_keywords or bool(matched_activation_keywords))
and all(matched_activation_keyword_groups)
)
public_item = {
key: value
for key, value in item.items()
@@ -939,6 +1024,9 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
"matched_keywords": matched,
"semantic_match": True,
"matched_activation_keywords": matched_activation_keywords,
"matched_activation_keyword_groups": (
matched_activation_keyword_groups
),
})
else:
unmatched.append(item["catalog_id"])
@@ -952,6 +1040,17 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
}
def get_ansible_catalog_candidates(
incident: dict[str, Any] | None,
drift: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
"""Return current semantically matched catalog rows for replay routing."""
hints = _catalog_hints(incident, drift)
candidates = hints.get("candidates")
return [dict(row) for row in candidates or [] if isinstance(row, dict)]
def build_ansible_truth(
automation_ops: list[dict[str, Any]],
*,

View File

@@ -19,7 +19,7 @@ from dataclasses import dataclass, replace
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from uuid import UUID, uuid4
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
import structlog
from sqlalchemy import select, text
@@ -31,7 +31,10 @@ from src.models.approval import ApprovalStatus
from src.services.ai_automation_runtime_contract import (
AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION,
)
from src.services.awooop_ansible_audit_service import get_ansible_catalog_item
from src.services.awooop_ansible_audit_service import (
get_ansible_catalog_candidates,
get_ansible_catalog_item,
)
from src.services.awooop_ansible_learning_writeback import (
canonical_ansible_playbook_id,
ensure_ansible_rag_writeback,
@@ -52,6 +55,7 @@ _CONTROLLED_RETRY_MAX_ATTEMPTS = 1
_CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS = 15
_CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS = 30
_STDIN_BOUNDARY_REPLAY_WINDOW_HOURS = 7 * 24
_CATALOG_REPAIR_REPLAY_WINDOW_HOURS = 72
_STDIN_BOUNDARY_TERMINAL_PROJECTION_BATCH_LIMIT = 5
_EXECUTION_CAPABILITY_MIN_TTL_SECONDS = 300
_EXECUTION_CAPABILITY_MAX_TTL_SECONDS = 1_800
@@ -1062,9 +1066,28 @@ def _claim_from_stale_check_mode_row(
or input_payload.get("incident_id")
or ""
)
catalog_id = str(input_payload.get("catalog_id") or "")
previous_catalog_id = str(input_payload.get("catalog_id") or "")
catalog_id = previous_catalog_id
incident_routing_evaluated = bool(
row.get("incident_alertname")
or row.get("incident_alert_category")
or row.get("incident_affected_services")
)
if incident_routing_evaluated:
candidates = get_ansible_catalog_candidates(
{
"alertname": row.get("incident_alertname"),
"alert_category": row.get("incident_alert_category"),
"affected_services": row.get("incident_affected_services") or [],
}
)
if not candidates:
return None
catalog_id = str(candidates[0].get("catalog_id") or "")
catalog_item = get_ansible_catalog_item(catalog_id) or {}
inventory_hosts = input_payload.get("inventory_hosts")
inventory_hosts = catalog_item.get("inventory_hosts") or input_payload.get(
"inventory_hosts"
)
source_playbook_path = str(
input_payload.get("source_candidate_playbook_path")
or input_payload.get("catalog_playbook_path")
@@ -1106,6 +1129,13 @@ def _claim_from_stale_check_mode_row(
)
except ValueError:
return None
claim_input.update(
{
"previous_catalog_id": previous_catalog_id or None,
"catalog_route_changed": catalog_id != previous_catalog_id,
"incident_semantic_route_rechecked": incident_routing_evaluated,
}
)
return AnsibleCheckModeClaim(
op_id=str(row.get("op_id") or ""),
source_candidate_op_id=source_candidate_op_id,
@@ -5663,6 +5693,10 @@ async def _send_controlled_apply_telegram_receipt(
try:
from src.services.telegram_gateway import get_telegram_gateway
effective_provider_delivery = str(
claim.input_payload.get("telegram_provider_delivery")
or provider_delivery
)
response = await get_telegram_gateway().send_controlled_apply_result_receipt(
automation_run_id=str(
claim.input_payload.get("automation_run_id")
@@ -5680,7 +5714,7 @@ async def _send_controlled_apply_telegram_receipt(
verifier_written=bool(writeback.get("verification")),
learning_written=bool(writeback.get("learning")),
execution_kind=execution_kind,
provider_delivery=provider_delivery,
provider_delivery=effective_provider_delivery,
project_id=project_id,
)
return bool(
@@ -7654,9 +7688,10 @@ async def claim_catalog_drift_failed_check_modes(
"""Replay a failed check once when its canonical check playbook changed."""
claims: list[AnsibleCheckModeClaim] = []
max_age_hours = (
max_age_hours = max(
candidate_max_age_hours
or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS,
_CATALOG_REPAIR_REPLAY_WINDOW_HOURS,
)
async with get_db_context(project_id) as db:
result = await db.execute(
@@ -7668,10 +7703,20 @@ async def claim_catalog_drift_failed_check_modes(
check_mode.input ->> 'incident_id',
check_mode.incident_id::text
) AS incident_id,
check_mode.input
check_mode.input,
incident.alertname AS incident_alertname,
incident.alert_category AS incident_alert_category,
incident.affected_services AS incident_affected_services
FROM automation_operation_log check_mode
JOIN incidents incident
ON incident.incident_id = coalesce(
check_mode.input ->> 'incident_id',
check_mode.incident_id::text
)
AND incident.project_id = :project_id
WHERE check_mode.operation_type = 'ansible_check_mode_executed'
AND check_mode.status = 'failed'
AND upper(incident.status::text) NOT IN ('RESOLVED', 'CLOSED')
AND check_mode.created_at >= NOW() - (
:candidate_max_age_hours * INTERVAL '1 hour'
)
@@ -7686,6 +7731,7 @@ async def claim_catalog_drift_failed_check_modes(
FOR UPDATE SKIP LOCKED
"""),
{
"project_id": project_id,
"candidate_max_age_hours": max(1, max_age_hours),
"scan_limit": min(100, max(10, limit * 10)),
},
@@ -7733,6 +7779,8 @@ async def claim_catalog_drift_failed_check_modes(
"replay_of_check_mode_op_id": str(row.get("op_id") or ""),
"previous_check_mode_playbook_path": previous_check_path,
"previous_catalog_revision": previous_catalog_revision or None,
"historical_projection_backfill": True,
"telegram_provider_delivery": "shadow_only",
}
inserted = await db.execute(
text("""
@@ -7761,6 +7809,9 @@ async def claim_catalog_drift_failed_check_modes(
= :catalog_replay_revision
AND existing_replay.input ->> 'catalog_drift_replay'
= 'true'
AND existing_replay.input
->> 'replay_of_check_mode_op_id'
= :failed_check_mode_op_id
)
RETURNING op_id
"""),
@@ -7786,6 +7837,7 @@ async def claim_catalog_drift_failed_check_modes(
"parent_op_id": canonical_claim.source_candidate_op_id,
"catalog_id": canonical_claim.catalog_id,
"catalog_replay_revision": catalog_replay_revision,
"failed_check_mode_op_id": str(row.get("op_id") or ""),
"tags": [
"ansible",
"check_mode",
@@ -7939,6 +7991,197 @@ async def claim_catalog_drift_failed_applies(
return claims
async def claim_semantically_misrouted_verified_applies(
*,
project_id: str = "awoooi",
limit: int = 1,
window_hours: int = _CATALOG_REPAIR_REPLAY_WINDOW_HOURS,
) -> list[AnsibleCheckModeClaim]:
"""Re-verify terminal applies whose catalog no longer matches the alert.
Resolved incidents remain eligible because this path corrects false terminal
attribution; it does not reopen incidents from an unresolved alert signal.
"""
claims: list[AnsibleCheckModeClaim] = []
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
source_check.op_id,
source_check.parent_op_id,
incident.incident_id,
source_check.input,
incident.alertname AS incident_alertname,
incident.alert_category AS incident_alert_category,
incident.affected_services AS incident_affected_services,
apply.op_id::text AS previous_apply_op_id
FROM automation_operation_log apply
JOIN automation_operation_log source_check
ON source_check.op_id = apply.parent_op_id
AND source_check.operation_type
= 'ansible_check_mode_executed'
JOIN incidents incident
ON incident.incident_id = coalesce(
apply.input ->> 'incident_id',
apply.incident_id::text
)
AND incident.project_id = :project_id
WHERE apply.operation_type = 'ansible_apply_executed'
AND apply.status = 'success'
AND apply.output ->> 'independent_post_verifier_passed'
= 'true'
AND apply.created_at >= NOW() - (
:window_hours * INTERVAL '1 hour'
)
ORDER BY apply.created_at DESC
LIMIT :scan_limit
FOR UPDATE OF apply SKIP LOCKED
"""),
{
"project_id": project_id,
"window_hours": max(1, min(window_hours, 7 * 24)),
"scan_limit": min(100, max(10, limit * 10)),
},
)
seen_incident_ids: set[str] = set()
for row_value in result.mappings().all():
row = dict(row_value)
incident_id = str(row.get("incident_id") or "")
if not incident_id or incident_id in seen_incident_ids:
continue
seen_incident_ids.add(incident_id)
previous_input = _json_loads(row.get("input"))
previous_catalog_id = str(
previous_input.get("catalog_id") or ""
)
canonical_claim = _claim_from_stale_check_mode_row(row)
if (
canonical_claim is None
or not previous_catalog_id
or canonical_claim.catalog_id == previous_catalog_id
):
continue
current_revision = str(
canonical_claim.input_payload.get("catalog_revision") or ""
)
previous_apply_op_id = str(
row.get("previous_apply_op_id") or ""
)
if not current_revision or not previous_apply_op_id:
continue
semantic_repair_run_id = str(
uuid5(
NAMESPACE_URL,
(
"awoooi:ansible-semantic-repair:"
f"{project_id}:{previous_apply_op_id}:"
f"{current_revision}"
),
)
)
replay_input = {
**canonical_claim.input_payload,
"automation_run_id": semantic_repair_run_id,
"semantic_route_reconciliation": True,
"semantic_repair_of_apply_op_id": previous_apply_op_id,
"catalog_replay_revision": current_revision,
"historical_projection_backfill": True,
"telegram_provider_delivery": "shadow_only",
"apply_idempotency_key": (
"ansible-semantic-repair:"
f"{previous_apply_op_id}:{current_revision}"
),
"bounded_execution_scope": (
"semantic_catalog_reconciliation_single_apply"
),
}
inserted = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
) SELECT
'ansible_check_mode_executed',
'ansible_check_mode_worker',
'pending',
:incident_db_id,
CAST(:input AS jsonb),
'{}'::jsonb,
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
WHERE NOT EXISTS (
SELECT 1
FROM automation_operation_log existing_replay
WHERE existing_replay.operation_type
= 'ansible_check_mode_executed'
AND existing_replay.input
->> 'semantic_repair_of_apply_op_id'
= :previous_apply_op_id
AND existing_replay.input
->> 'catalog_replay_revision'
= :catalog_replay_revision
)
RETURNING op_id
"""),
{
"incident_db_id": _automation_operation_log_incident_id(
canonical_claim.incident_id
),
"input": json.dumps(replay_input, ensure_ascii=False),
"dry_run_result": json.dumps(
{
"check_mode_executed": False,
"apply_executed": False,
"claim_state": (
"semantic_catalog_reconciliation"
),
"controlled_apply_allowed": bool(
replay_input.get("controlled_apply_allowed")
),
},
ensure_ascii=False,
),
"parent_op_id": (
canonical_claim.source_candidate_op_id
),
"previous_apply_op_id": previous_apply_op_id,
"catalog_replay_revision": current_revision,
"tags": [
"ansible",
"check_mode",
"pending",
"semantic_catalog_reconciliation",
],
},
)
op_id = inserted.scalar_one_or_none()
if op_id is None:
continue
claims.append(
AnsibleCheckModeClaim(
op_id=str(op_id),
source_candidate_op_id=(
canonical_claim.source_candidate_op_id
),
incident_id=canonical_claim.incident_id,
catalog_id=canonical_claim.catalog_id,
playbook_path=canonical_claim.playbook_path,
apply_playbook_path=(
canonical_claim.apply_playbook_path
),
inventory_hosts=canonical_claim.inventory_hosts,
risk_level=canonical_claim.risk_level,
input_payload=replay_input,
)
)
if len(claims) >= max(1, limit):
break
return claims
async def recent_ansible_transport_blockers(
*,
project_id: str = "awoooi",
@@ -8716,6 +8959,30 @@ async def run_pending_check_modes_once(
0,
remaining_limit - len(stdin_boundary_replay_claims),
)
semantic_catalog_reconciliation_claims: list[
AnsibleCheckModeClaim
] = []
semantic_catalog_reconciliation_error: str | None = None
if remaining_limit:
try:
semantic_catalog_reconciliation_claims = (
await claim_semantically_misrouted_verified_applies(
project_id=project_id,
limit=remaining_limit,
)
)
except Exception as exc:
semantic_catalog_reconciliation_error = type(exc).__name__
logger.warning(
"ansible_semantic_catalog_reconciliation_claim_failed",
project_id=project_id,
error_type=semantic_catalog_reconciliation_error,
fresh_candidate_claim_continues=True,
)
remaining_limit = max(
0,
remaining_limit - len(semantic_catalog_reconciliation_claims),
)
failed_apply_catalog_replay_claims: list[AnsibleCheckModeClaim] = []
failed_apply_catalog_replay_error: str | None = None
if remaining_limit:
@@ -8775,6 +9042,7 @@ async def run_pending_check_modes_once(
claims = [
*reclaimed_claims,
*stdin_boundary_replay_claims,
*semantic_catalog_reconciliation_claims,
*failed_apply_catalog_replay_claims,
*catalog_replay_claims,
*fresh_claims,
@@ -8924,6 +9192,12 @@ async def run_pending_check_modes_once(
stdin_boundary_replay_terminal_written
),
"stdin_boundary_replay_error": stdin_boundary_replay_error,
"semantic_catalog_reconciled": len(
semantic_catalog_reconciliation_claims
),
"semantic_catalog_reconciliation_error": (
semantic_catalog_reconciliation_error
),
"failed_apply_catalog_replayed": len(
failed_apply_catalog_replay_claims
),

View File

@@ -98,6 +98,36 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = {
"test -s /home/wooo/node_exporter_textfiles/docker_stats.prom",
),
),
"ansible:110-disk-pressure": (
AssetPostcondition(
"host_110_root_disk_below_alert_threshold",
"metric",
"host_110",
"usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); "
"test -n \"$usage\" && test \"$usage\" -lt 85",
),
AssetPostcondition(
"host_110_docker_runtime_after_cleanup",
"service",
"host_110",
"docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0",
),
),
"ansible:188-disk-pressure": (
AssetPostcondition(
"host_188_root_disk_below_alert_threshold",
"metric",
"host_188",
"usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); "
"test -n \"$usage\" && test \"$usage\" -lt 85",
),
AssetPostcondition(
"host_188_docker_runtime_after_cleanup",
"service",
"host_188",
"docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0",
),
),
"ansible:110-devops": (
AssetPostcondition(
"host_110_node_exporter_container",

View File

@@ -5312,9 +5312,9 @@ class TelegramGateway:
row = existing.mappings().one_or_none()
if row is not None:
existing_row = dict(row)
elif failure_fingerprint:
elif force_suppress or failure_fingerprint:
should_suppress = force_suppress
if not should_suppress:
if not should_suppress and failure_fingerprint:
group_reservation = await db.execute(
text("""
SELECT message_id::text AS message_id
@@ -10370,6 +10370,20 @@ class TelegramGateway:
"km_rag_playbook",
],
}
elif provider_delivery == "shadow_only":
source_extra["notification_policy"] = {
"policy_version": "telegram_historical_repair_shadow_v1",
"failure_fingerprint": "",
"group_scope": "historical_projection_backfill",
"provider_delivery": "shadow_only",
"disposition": "suppressed",
"provider_send_performed": False,
"details_retained_in": [
"automation_operation_log",
"awooop_runs",
"km_rag_playbook",
],
}
if no_write_replay:
lines = [
f"<b>{html.escape(title)}</b>",