fix(agent): route disk pressure to bounded repair
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
This commit is contained in:
@@ -417,6 +417,11 @@ _CATALOG: tuple[dict[str, Any], ...] = (
|
||||
"gitea_service",
|
||||
"systemd_control_plane",
|
||||
],
|
||||
"activation_keywords": [
|
||||
"host110sustainedmoderatepressure",
|
||||
"host-pressure-controller",
|
||||
"sustained pressure",
|
||||
],
|
||||
"supports_check_mode": True,
|
||||
"auto_apply_enabled": False,
|
||||
"approval_required": False,
|
||||
@@ -424,6 +429,74 @@ _CATALOG: tuple[dict[str, Any], ...] = (
|
||||
"no_write_only": True,
|
||||
"catalog_revision": "2026-07-14-host110-inputs-v2",
|
||||
},
|
||||
{
|
||||
"catalog_id": "ansible:110-disk-pressure",
|
||||
"playbook_path": (
|
||||
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
|
||||
),
|
||||
"check_mode_playbook_path": (
|
||||
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
|
||||
),
|
||||
"inventory_hosts": ["host_110"],
|
||||
"domains": ["disk_pressure", "docker_artifacts", "host_110"],
|
||||
"keywords": [
|
||||
"hostoutofdiskspace",
|
||||
"hostdiskusagehigh",
|
||||
"hostdiskusagecritical",
|
||||
"hostdiskwillfillin24hours",
|
||||
"disk_full",
|
||||
"node-exporter-110",
|
||||
"192.168.0.110",
|
||||
],
|
||||
"activation_keyword_groups": [
|
||||
[
|
||||
"hostoutofdiskspace",
|
||||
"hostdiskusagehigh",
|
||||
"hostdiskusagecritical",
|
||||
"hostdiskwillfillin24hours",
|
||||
],
|
||||
["node-exporter-110", "192.168.0.110", "host_110", "host110"],
|
||||
],
|
||||
"supports_check_mode": True,
|
||||
"auto_apply_enabled": True,
|
||||
"approval_required": False,
|
||||
"risk_level": "medium",
|
||||
"catalog_revision": "2026-07-14-disk-pressure-bounded-v1",
|
||||
},
|
||||
{
|
||||
"catalog_id": "ansible:188-disk-pressure",
|
||||
"playbook_path": (
|
||||
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
|
||||
),
|
||||
"check_mode_playbook_path": (
|
||||
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
|
||||
),
|
||||
"inventory_hosts": ["host_188"],
|
||||
"domains": ["disk_pressure", "docker_artifacts", "host_188"],
|
||||
"keywords": [
|
||||
"hostoutofdiskspace",
|
||||
"hostdiskusagehigh",
|
||||
"hostdiskusagecritical",
|
||||
"hostdiskwillfillin24hours",
|
||||
"disk_full",
|
||||
"node-exporter-188",
|
||||
"192.168.0.188",
|
||||
],
|
||||
"activation_keyword_groups": [
|
||||
[
|
||||
"hostoutofdiskspace",
|
||||
"hostdiskusagehigh",
|
||||
"hostdiskusagecritical",
|
||||
"hostdiskwillfillin24hours",
|
||||
],
|
||||
["node-exporter-188", "192.168.0.188", "host_188", "host188"],
|
||||
],
|
||||
"supports_check_mode": True,
|
||||
"auto_apply_enabled": True,
|
||||
"approval_required": False,
|
||||
"risk_level": "medium",
|
||||
"catalog_revision": "2026-07-14-disk-pressure-bounded-v1",
|
||||
},
|
||||
{
|
||||
"catalog_id": "ansible:110-devops",
|
||||
"playbook_path": "infra/ansible/playbooks/110-node-exporter-repair.yml",
|
||||
@@ -437,9 +510,13 @@ _CATALOG: tuple[dict[str, Any], ...] = (
|
||||
"node_exporter",
|
||||
"node-exporter",
|
||||
],
|
||||
"activation_keywords": [
|
||||
"node-exporter-110",
|
||||
"node exporter 110",
|
||||
"activation_keyword_groups": [
|
||||
[
|
||||
"nodeexporterdown",
|
||||
"nodeexporterscrapedown",
|
||||
"node_exporter_down",
|
||||
],
|
||||
["node-exporter-110", "192.168.0.110", "host_110", "host110"],
|
||||
],
|
||||
"supports_check_mode": True,
|
||||
"auto_apply_enabled": True,
|
||||
@@ -914,7 +991,15 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
|
||||
matched_activation_keywords = [
|
||||
keyword for keyword in activation_keywords if keyword in haystack
|
||||
]
|
||||
semantic_match = not activation_keywords or bool(matched_activation_keywords)
|
||||
activation_keyword_groups = item.get("activation_keyword_groups") or []
|
||||
matched_activation_keyword_groups = [
|
||||
[keyword for keyword in group if keyword in haystack]
|
||||
for group in activation_keyword_groups
|
||||
]
|
||||
semantic_match = (
|
||||
(not activation_keywords or bool(matched_activation_keywords))
|
||||
and all(matched_activation_keyword_groups)
|
||||
)
|
||||
public_item = {
|
||||
key: value
|
||||
for key, value in item.items()
|
||||
@@ -939,6 +1024,9 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
|
||||
"matched_keywords": matched,
|
||||
"semantic_match": True,
|
||||
"matched_activation_keywords": matched_activation_keywords,
|
||||
"matched_activation_keyword_groups": (
|
||||
matched_activation_keyword_groups
|
||||
),
|
||||
})
|
||||
else:
|
||||
unmatched.append(item["catalog_id"])
|
||||
@@ -952,6 +1040,17 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
|
||||
}
|
||||
|
||||
|
||||
def get_ansible_catalog_candidates(
|
||||
incident: dict[str, Any] | None,
|
||||
drift: dict[str, Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return current semantically matched catalog rows for replay routing."""
|
||||
|
||||
hints = _catalog_hints(incident, drift)
|
||||
candidates = hints.get("candidates")
|
||||
return [dict(row) for row in candidates or [] if isinstance(row, dict)]
|
||||
|
||||
|
||||
def build_ansible_truth(
|
||||
automation_ops: list[dict[str, Any]],
|
||||
*,
|
||||
|
||||
@@ -19,7 +19,7 @@ from dataclasses import dataclass, replace
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
|
||||
|
||||
import structlog
|
||||
from sqlalchemy import select, text
|
||||
@@ -31,7 +31,10 @@ from src.models.approval import ApprovalStatus
|
||||
from src.services.ai_automation_runtime_contract import (
|
||||
AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION,
|
||||
)
|
||||
from src.services.awooop_ansible_audit_service import get_ansible_catalog_item
|
||||
from src.services.awooop_ansible_audit_service import (
|
||||
get_ansible_catalog_candidates,
|
||||
get_ansible_catalog_item,
|
||||
)
|
||||
from src.services.awooop_ansible_learning_writeback import (
|
||||
canonical_ansible_playbook_id,
|
||||
ensure_ansible_rag_writeback,
|
||||
@@ -52,6 +55,7 @@ _CONTROLLED_RETRY_MAX_ATTEMPTS = 1
|
||||
_CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS = 15
|
||||
_CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS = 30
|
||||
_STDIN_BOUNDARY_REPLAY_WINDOW_HOURS = 7 * 24
|
||||
_CATALOG_REPAIR_REPLAY_WINDOW_HOURS = 72
|
||||
_STDIN_BOUNDARY_TERMINAL_PROJECTION_BATCH_LIMIT = 5
|
||||
_EXECUTION_CAPABILITY_MIN_TTL_SECONDS = 300
|
||||
_EXECUTION_CAPABILITY_MAX_TTL_SECONDS = 1_800
|
||||
@@ -1062,9 +1066,28 @@ def _claim_from_stale_check_mode_row(
|
||||
or input_payload.get("incident_id")
|
||||
or ""
|
||||
)
|
||||
catalog_id = str(input_payload.get("catalog_id") or "")
|
||||
previous_catalog_id = str(input_payload.get("catalog_id") or "")
|
||||
catalog_id = previous_catalog_id
|
||||
incident_routing_evaluated = bool(
|
||||
row.get("incident_alertname")
|
||||
or row.get("incident_alert_category")
|
||||
or row.get("incident_affected_services")
|
||||
)
|
||||
if incident_routing_evaluated:
|
||||
candidates = get_ansible_catalog_candidates(
|
||||
{
|
||||
"alertname": row.get("incident_alertname"),
|
||||
"alert_category": row.get("incident_alert_category"),
|
||||
"affected_services": row.get("incident_affected_services") or [],
|
||||
}
|
||||
)
|
||||
if not candidates:
|
||||
return None
|
||||
catalog_id = str(candidates[0].get("catalog_id") or "")
|
||||
catalog_item = get_ansible_catalog_item(catalog_id) or {}
|
||||
inventory_hosts = input_payload.get("inventory_hosts")
|
||||
inventory_hosts = catalog_item.get("inventory_hosts") or input_payload.get(
|
||||
"inventory_hosts"
|
||||
)
|
||||
source_playbook_path = str(
|
||||
input_payload.get("source_candidate_playbook_path")
|
||||
or input_payload.get("catalog_playbook_path")
|
||||
@@ -1106,6 +1129,13 @@ def _claim_from_stale_check_mode_row(
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
claim_input.update(
|
||||
{
|
||||
"previous_catalog_id": previous_catalog_id or None,
|
||||
"catalog_route_changed": catalog_id != previous_catalog_id,
|
||||
"incident_semantic_route_rechecked": incident_routing_evaluated,
|
||||
}
|
||||
)
|
||||
return AnsibleCheckModeClaim(
|
||||
op_id=str(row.get("op_id") or ""),
|
||||
source_candidate_op_id=source_candidate_op_id,
|
||||
@@ -5663,6 +5693,10 @@ async def _send_controlled_apply_telegram_receipt(
|
||||
try:
|
||||
from src.services.telegram_gateway import get_telegram_gateway
|
||||
|
||||
effective_provider_delivery = str(
|
||||
claim.input_payload.get("telegram_provider_delivery")
|
||||
or provider_delivery
|
||||
)
|
||||
response = await get_telegram_gateway().send_controlled_apply_result_receipt(
|
||||
automation_run_id=str(
|
||||
claim.input_payload.get("automation_run_id")
|
||||
@@ -5680,7 +5714,7 @@ async def _send_controlled_apply_telegram_receipt(
|
||||
verifier_written=bool(writeback.get("verification")),
|
||||
learning_written=bool(writeback.get("learning")),
|
||||
execution_kind=execution_kind,
|
||||
provider_delivery=provider_delivery,
|
||||
provider_delivery=effective_provider_delivery,
|
||||
project_id=project_id,
|
||||
)
|
||||
return bool(
|
||||
@@ -7654,9 +7688,10 @@ async def claim_catalog_drift_failed_check_modes(
|
||||
"""Replay a failed check once when its canonical check playbook changed."""
|
||||
|
||||
claims: list[AnsibleCheckModeClaim] = []
|
||||
max_age_hours = (
|
||||
max_age_hours = max(
|
||||
candidate_max_age_hours
|
||||
or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
|
||||
or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS,
|
||||
_CATALOG_REPAIR_REPLAY_WINDOW_HOURS,
|
||||
)
|
||||
async with get_db_context(project_id) as db:
|
||||
result = await db.execute(
|
||||
@@ -7668,10 +7703,20 @@ async def claim_catalog_drift_failed_check_modes(
|
||||
check_mode.input ->> 'incident_id',
|
||||
check_mode.incident_id::text
|
||||
) AS incident_id,
|
||||
check_mode.input
|
||||
check_mode.input,
|
||||
incident.alertname AS incident_alertname,
|
||||
incident.alert_category AS incident_alert_category,
|
||||
incident.affected_services AS incident_affected_services
|
||||
FROM automation_operation_log check_mode
|
||||
JOIN incidents incident
|
||||
ON incident.incident_id = coalesce(
|
||||
check_mode.input ->> 'incident_id',
|
||||
check_mode.incident_id::text
|
||||
)
|
||||
AND incident.project_id = :project_id
|
||||
WHERE check_mode.operation_type = 'ansible_check_mode_executed'
|
||||
AND check_mode.status = 'failed'
|
||||
AND upper(incident.status::text) NOT IN ('RESOLVED', 'CLOSED')
|
||||
AND check_mode.created_at >= NOW() - (
|
||||
:candidate_max_age_hours * INTERVAL '1 hour'
|
||||
)
|
||||
@@ -7686,6 +7731,7 @@ async def claim_catalog_drift_failed_check_modes(
|
||||
FOR UPDATE SKIP LOCKED
|
||||
"""),
|
||||
{
|
||||
"project_id": project_id,
|
||||
"candidate_max_age_hours": max(1, max_age_hours),
|
||||
"scan_limit": min(100, max(10, limit * 10)),
|
||||
},
|
||||
@@ -7733,6 +7779,8 @@ async def claim_catalog_drift_failed_check_modes(
|
||||
"replay_of_check_mode_op_id": str(row.get("op_id") or ""),
|
||||
"previous_check_mode_playbook_path": previous_check_path,
|
||||
"previous_catalog_revision": previous_catalog_revision or None,
|
||||
"historical_projection_backfill": True,
|
||||
"telegram_provider_delivery": "shadow_only",
|
||||
}
|
||||
inserted = await db.execute(
|
||||
text("""
|
||||
@@ -7761,6 +7809,9 @@ async def claim_catalog_drift_failed_check_modes(
|
||||
= :catalog_replay_revision
|
||||
AND existing_replay.input ->> 'catalog_drift_replay'
|
||||
= 'true'
|
||||
AND existing_replay.input
|
||||
->> 'replay_of_check_mode_op_id'
|
||||
= :failed_check_mode_op_id
|
||||
)
|
||||
RETURNING op_id
|
||||
"""),
|
||||
@@ -7786,6 +7837,7 @@ async def claim_catalog_drift_failed_check_modes(
|
||||
"parent_op_id": canonical_claim.source_candidate_op_id,
|
||||
"catalog_id": canonical_claim.catalog_id,
|
||||
"catalog_replay_revision": catalog_replay_revision,
|
||||
"failed_check_mode_op_id": str(row.get("op_id") or ""),
|
||||
"tags": [
|
||||
"ansible",
|
||||
"check_mode",
|
||||
@@ -7939,6 +7991,197 @@ async def claim_catalog_drift_failed_applies(
|
||||
return claims
|
||||
|
||||
|
||||
async def claim_semantically_misrouted_verified_applies(
|
||||
*,
|
||||
project_id: str = "awoooi",
|
||||
limit: int = 1,
|
||||
window_hours: int = _CATALOG_REPAIR_REPLAY_WINDOW_HOURS,
|
||||
) -> list[AnsibleCheckModeClaim]:
|
||||
"""Re-verify terminal applies whose catalog no longer matches the alert.
|
||||
|
||||
Resolved incidents remain eligible because this path corrects false terminal
|
||||
attribution; it does not reopen incidents from an unresolved alert signal.
|
||||
"""
|
||||
|
||||
claims: list[AnsibleCheckModeClaim] = []
|
||||
async with get_db_context(project_id) as db:
|
||||
result = await db.execute(
|
||||
text("""
|
||||
SELECT
|
||||
source_check.op_id,
|
||||
source_check.parent_op_id,
|
||||
incident.incident_id,
|
||||
source_check.input,
|
||||
incident.alertname AS incident_alertname,
|
||||
incident.alert_category AS incident_alert_category,
|
||||
incident.affected_services AS incident_affected_services,
|
||||
apply.op_id::text AS previous_apply_op_id
|
||||
FROM automation_operation_log apply
|
||||
JOIN automation_operation_log source_check
|
||||
ON source_check.op_id = apply.parent_op_id
|
||||
AND source_check.operation_type
|
||||
= 'ansible_check_mode_executed'
|
||||
JOIN incidents incident
|
||||
ON incident.incident_id = coalesce(
|
||||
apply.input ->> 'incident_id',
|
||||
apply.incident_id::text
|
||||
)
|
||||
AND incident.project_id = :project_id
|
||||
WHERE apply.operation_type = 'ansible_apply_executed'
|
||||
AND apply.status = 'success'
|
||||
AND apply.output ->> 'independent_post_verifier_passed'
|
||||
= 'true'
|
||||
AND apply.created_at >= NOW() - (
|
||||
:window_hours * INTERVAL '1 hour'
|
||||
)
|
||||
ORDER BY apply.created_at DESC
|
||||
LIMIT :scan_limit
|
||||
FOR UPDATE OF apply SKIP LOCKED
|
||||
"""),
|
||||
{
|
||||
"project_id": project_id,
|
||||
"window_hours": max(1, min(window_hours, 7 * 24)),
|
||||
"scan_limit": min(100, max(10, limit * 10)),
|
||||
},
|
||||
)
|
||||
seen_incident_ids: set[str] = set()
|
||||
for row_value in result.mappings().all():
|
||||
row = dict(row_value)
|
||||
incident_id = str(row.get("incident_id") or "")
|
||||
if not incident_id or incident_id in seen_incident_ids:
|
||||
continue
|
||||
seen_incident_ids.add(incident_id)
|
||||
previous_input = _json_loads(row.get("input"))
|
||||
previous_catalog_id = str(
|
||||
previous_input.get("catalog_id") or ""
|
||||
)
|
||||
canonical_claim = _claim_from_stale_check_mode_row(row)
|
||||
if (
|
||||
canonical_claim is None
|
||||
or not previous_catalog_id
|
||||
or canonical_claim.catalog_id == previous_catalog_id
|
||||
):
|
||||
continue
|
||||
current_revision = str(
|
||||
canonical_claim.input_payload.get("catalog_revision") or ""
|
||||
)
|
||||
previous_apply_op_id = str(
|
||||
row.get("previous_apply_op_id") or ""
|
||||
)
|
||||
if not current_revision or not previous_apply_op_id:
|
||||
continue
|
||||
semantic_repair_run_id = str(
|
||||
uuid5(
|
||||
NAMESPACE_URL,
|
||||
(
|
||||
"awoooi:ansible-semantic-repair:"
|
||||
f"{project_id}:{previous_apply_op_id}:"
|
||||
f"{current_revision}"
|
||||
),
|
||||
)
|
||||
)
|
||||
replay_input = {
|
||||
**canonical_claim.input_payload,
|
||||
"automation_run_id": semantic_repair_run_id,
|
||||
"semantic_route_reconciliation": True,
|
||||
"semantic_repair_of_apply_op_id": previous_apply_op_id,
|
||||
"catalog_replay_revision": current_revision,
|
||||
"historical_projection_backfill": True,
|
||||
"telegram_provider_delivery": "shadow_only",
|
||||
"apply_idempotency_key": (
|
||||
"ansible-semantic-repair:"
|
||||
f"{previous_apply_op_id}:{current_revision}"
|
||||
),
|
||||
"bounded_execution_scope": (
|
||||
"semantic_catalog_reconciliation_single_apply"
|
||||
),
|
||||
}
|
||||
inserted = await db.execute(
|
||||
text("""
|
||||
INSERT INTO automation_operation_log (
|
||||
operation_type, actor, status, incident_id,
|
||||
input, output, dry_run_result,
|
||||
parent_op_id, tags
|
||||
) SELECT
|
||||
'ansible_check_mode_executed',
|
||||
'ansible_check_mode_worker',
|
||||
'pending',
|
||||
:incident_db_id,
|
||||
CAST(:input AS jsonb),
|
||||
'{}'::jsonb,
|
||||
CAST(:dry_run_result AS jsonb),
|
||||
CAST(:parent_op_id AS uuid),
|
||||
:tags
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM automation_operation_log existing_replay
|
||||
WHERE existing_replay.operation_type
|
||||
= 'ansible_check_mode_executed'
|
||||
AND existing_replay.input
|
||||
->> 'semantic_repair_of_apply_op_id'
|
||||
= :previous_apply_op_id
|
||||
AND existing_replay.input
|
||||
->> 'catalog_replay_revision'
|
||||
= :catalog_replay_revision
|
||||
)
|
||||
RETURNING op_id
|
||||
"""),
|
||||
{
|
||||
"incident_db_id": _automation_operation_log_incident_id(
|
||||
canonical_claim.incident_id
|
||||
),
|
||||
"input": json.dumps(replay_input, ensure_ascii=False),
|
||||
"dry_run_result": json.dumps(
|
||||
{
|
||||
"check_mode_executed": False,
|
||||
"apply_executed": False,
|
||||
"claim_state": (
|
||||
"semantic_catalog_reconciliation"
|
||||
),
|
||||
"controlled_apply_allowed": bool(
|
||||
replay_input.get("controlled_apply_allowed")
|
||||
),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
"parent_op_id": (
|
||||
canonical_claim.source_candidate_op_id
|
||||
),
|
||||
"previous_apply_op_id": previous_apply_op_id,
|
||||
"catalog_replay_revision": current_revision,
|
||||
"tags": [
|
||||
"ansible",
|
||||
"check_mode",
|
||||
"pending",
|
||||
"semantic_catalog_reconciliation",
|
||||
],
|
||||
},
|
||||
)
|
||||
op_id = inserted.scalar_one_or_none()
|
||||
if op_id is None:
|
||||
continue
|
||||
claims.append(
|
||||
AnsibleCheckModeClaim(
|
||||
op_id=str(op_id),
|
||||
source_candidate_op_id=(
|
||||
canonical_claim.source_candidate_op_id
|
||||
),
|
||||
incident_id=canonical_claim.incident_id,
|
||||
catalog_id=canonical_claim.catalog_id,
|
||||
playbook_path=canonical_claim.playbook_path,
|
||||
apply_playbook_path=(
|
||||
canonical_claim.apply_playbook_path
|
||||
),
|
||||
inventory_hosts=canonical_claim.inventory_hosts,
|
||||
risk_level=canonical_claim.risk_level,
|
||||
input_payload=replay_input,
|
||||
)
|
||||
)
|
||||
if len(claims) >= max(1, limit):
|
||||
break
|
||||
return claims
|
||||
|
||||
|
||||
async def recent_ansible_transport_blockers(
|
||||
*,
|
||||
project_id: str = "awoooi",
|
||||
@@ -8716,6 +8959,30 @@ async def run_pending_check_modes_once(
|
||||
0,
|
||||
remaining_limit - len(stdin_boundary_replay_claims),
|
||||
)
|
||||
semantic_catalog_reconciliation_claims: list[
|
||||
AnsibleCheckModeClaim
|
||||
] = []
|
||||
semantic_catalog_reconciliation_error: str | None = None
|
||||
if remaining_limit:
|
||||
try:
|
||||
semantic_catalog_reconciliation_claims = (
|
||||
await claim_semantically_misrouted_verified_applies(
|
||||
project_id=project_id,
|
||||
limit=remaining_limit,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
semantic_catalog_reconciliation_error = type(exc).__name__
|
||||
logger.warning(
|
||||
"ansible_semantic_catalog_reconciliation_claim_failed",
|
||||
project_id=project_id,
|
||||
error_type=semantic_catalog_reconciliation_error,
|
||||
fresh_candidate_claim_continues=True,
|
||||
)
|
||||
remaining_limit = max(
|
||||
0,
|
||||
remaining_limit - len(semantic_catalog_reconciliation_claims),
|
||||
)
|
||||
failed_apply_catalog_replay_claims: list[AnsibleCheckModeClaim] = []
|
||||
failed_apply_catalog_replay_error: str | None = None
|
||||
if remaining_limit:
|
||||
@@ -8775,6 +9042,7 @@ async def run_pending_check_modes_once(
|
||||
claims = [
|
||||
*reclaimed_claims,
|
||||
*stdin_boundary_replay_claims,
|
||||
*semantic_catalog_reconciliation_claims,
|
||||
*failed_apply_catalog_replay_claims,
|
||||
*catalog_replay_claims,
|
||||
*fresh_claims,
|
||||
@@ -8924,6 +9192,12 @@ async def run_pending_check_modes_once(
|
||||
stdin_boundary_replay_terminal_written
|
||||
),
|
||||
"stdin_boundary_replay_error": stdin_boundary_replay_error,
|
||||
"semantic_catalog_reconciled": len(
|
||||
semantic_catalog_reconciliation_claims
|
||||
),
|
||||
"semantic_catalog_reconciliation_error": (
|
||||
semantic_catalog_reconciliation_error
|
||||
),
|
||||
"failed_apply_catalog_replayed": len(
|
||||
failed_apply_catalog_replay_claims
|
||||
),
|
||||
|
||||
@@ -98,6 +98,36 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = {
|
||||
"test -s /home/wooo/node_exporter_textfiles/docker_stats.prom",
|
||||
),
|
||||
),
|
||||
"ansible:110-disk-pressure": (
|
||||
AssetPostcondition(
|
||||
"host_110_root_disk_below_alert_threshold",
|
||||
"metric",
|
||||
"host_110",
|
||||
"usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); "
|
||||
"test -n \"$usage\" && test \"$usage\" -lt 85",
|
||||
),
|
||||
AssetPostcondition(
|
||||
"host_110_docker_runtime_after_cleanup",
|
||||
"service",
|
||||
"host_110",
|
||||
"docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0",
|
||||
),
|
||||
),
|
||||
"ansible:188-disk-pressure": (
|
||||
AssetPostcondition(
|
||||
"host_188_root_disk_below_alert_threshold",
|
||||
"metric",
|
||||
"host_188",
|
||||
"usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); "
|
||||
"test -n \"$usage\" && test \"$usage\" -lt 85",
|
||||
),
|
||||
AssetPostcondition(
|
||||
"host_188_docker_runtime_after_cleanup",
|
||||
"service",
|
||||
"host_188",
|
||||
"docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0",
|
||||
),
|
||||
),
|
||||
"ansible:110-devops": (
|
||||
AssetPostcondition(
|
||||
"host_110_node_exporter_container",
|
||||
|
||||
@@ -5312,9 +5312,9 @@ class TelegramGateway:
|
||||
row = existing.mappings().one_or_none()
|
||||
if row is not None:
|
||||
existing_row = dict(row)
|
||||
elif failure_fingerprint:
|
||||
elif force_suppress or failure_fingerprint:
|
||||
should_suppress = force_suppress
|
||||
if not should_suppress:
|
||||
if not should_suppress and failure_fingerprint:
|
||||
group_reservation = await db.execute(
|
||||
text("""
|
||||
SELECT message_id::text AS message_id
|
||||
@@ -10370,6 +10370,20 @@ class TelegramGateway:
|
||||
"km_rag_playbook",
|
||||
],
|
||||
}
|
||||
elif provider_delivery == "shadow_only":
|
||||
source_extra["notification_policy"] = {
|
||||
"policy_version": "telegram_historical_repair_shadow_v1",
|
||||
"failure_fingerprint": "",
|
||||
"group_scope": "historical_projection_backfill",
|
||||
"provider_delivery": "shadow_only",
|
||||
"disposition": "suppressed",
|
||||
"provider_send_performed": False,
|
||||
"details_retained_in": [
|
||||
"automation_operation_log",
|
||||
"awooop_runs",
|
||||
"km_rag_playbook",
|
||||
],
|
||||
}
|
||||
if no_write_replay:
|
||||
lines = [
|
||||
f"<b>{html.escape(title)}</b>",
|
||||
|
||||
Reference in New Issue
Block a user