fix(ai): improve docker repair verification signals
This commit is contained in:
@@ -255,6 +255,8 @@ class PostExecutionVerifier:
|
||||
"pod_name": labels.get("pod", labels.get("name", "")),
|
||||
"deployment": labels.get("deployment", ""),
|
||||
"host": labels.get("instance", "").split(":")[0] or labels.get("host", ""),
|
||||
"container_name": _extract_container_name(labels),
|
||||
"filter_name": _extract_container_name(labels),
|
||||
"query": _build_prometheus_query(alertname, labels),
|
||||
}
|
||||
|
||||
@@ -495,7 +497,13 @@ def _assess_recovery(
|
||||
"'success': true",
|
||||
'"success": true',
|
||||
]
|
||||
if any(sig in post_str for sig in success_signals):
|
||||
has_success_signal = any(sig in post_str for sig in success_signals)
|
||||
if not has_success_signal and _docker_state_indicates_running(post_str):
|
||||
has_success_signal = True
|
||||
|
||||
if has_success_signal:
|
||||
if _is_observe_only_action(action_taken):
|
||||
return "degraded"
|
||||
# 但如果 pre_state 已經是 running,可能是無效操作
|
||||
if pre_str and any(sig in pre_str for sig in success_signals):
|
||||
# 如果執行的是 restart,即使 pre/post 都 Running 也算 success
|
||||
@@ -533,6 +541,53 @@ def _get_labels(incident: "Incident") -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
def _extract_container_name(labels: dict[str, Any]) -> str:
|
||||
"""Resolve Docker container target labels for post-execution SSH sensors."""
|
||||
for key in ("filter_name", "container_name", "container", "resource", "name"):
|
||||
value = str(labels.get(key) or "").strip()
|
||||
if value and "{" not in value and "}" not in value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def _is_observe_only_action(action_taken: str) -> bool:
|
||||
"""Return true when the executed step collected evidence but did not mutate state."""
|
||||
lowered = (action_taken or "").lower()
|
||||
observe_tokens = (
|
||||
"mcp:ssh_diagnose",
|
||||
"ssh_diagnose",
|
||||
"docker stats",
|
||||
"ps aux",
|
||||
"free -h",
|
||||
"df -h",
|
||||
)
|
||||
mutation_tokens = (
|
||||
"restart",
|
||||
"delete",
|
||||
"rollout",
|
||||
"scale",
|
||||
"patch",
|
||||
"apply",
|
||||
"prune",
|
||||
"truncate",
|
||||
"clear",
|
||||
)
|
||||
return any(token in lowered for token in observe_tokens) and not any(
|
||||
token in lowered for token in mutation_tokens
|
||||
)
|
||||
|
||||
|
||||
def _docker_state_indicates_running(post_str: str) -> bool:
|
||||
"""Recognize Docker-specific healthy/running output without matching uptime."""
|
||||
if not any(token in post_str for token in ("docker ps", "docker inspect", "docker stats")):
|
||||
return False
|
||||
return bool(
|
||||
re.search(r'\bup\s+\d', post_str)
|
||||
or re.search(r'["\']status["\']\s*:\s*["\']running["\']', post_str)
|
||||
or re.search(r'["\']running["\']\s*:\s*true', post_str)
|
||||
)
|
||||
|
||||
|
||||
def _build_prometheus_query(alertname: str, labels: dict[str, Any]) -> str:
|
||||
"""Build a non-empty PromQL probe for post-execution metric sensors."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user