From e3acc321a75d3c6c669acbd80cf8fec0c5e84ad2 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 02:19:52 +0800 Subject: [PATCH 1/5] fix(agent): retain shadow policy on stale replay --- .../awooop_ansible_check_mode_service.py | 33 ++++++++++++++ ...test_host_disk_pressure_ansible_catalog.py | 44 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 9edb2b681..8439a5716 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -56,6 +56,20 @@ _CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS = 15 _CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS = 30 _STDIN_BOUNDARY_REPLAY_WINDOW_HOURS = 7 * 24 _CATALOG_REPAIR_REPLAY_WINDOW_HOURS = 72 +_STALE_CLAIM_REPLAY_CONTROL_FIELDS = ( + "bounded_execution_scope", + "catalog_drift_replay", + "catalog_repair_of_apply_op_id", + "catalog_replay_revision", + "previous_catalog_revision", + "previous_check_mode_playbook_path", + "replay_of_check_mode_op_id", + "semantic_repair_of_apply_op_id", + "semantic_route_reconciliation", +) +_HISTORICAL_SHADOW_REPLAY_TAGS = frozenset( + {"catalog_drift_replay", "semantic_catalog_reconciliation"} +) _STDIN_BOUNDARY_TERMINAL_PROJECTION_BATCH_LIMIT = 5 _EXECUTION_CAPABILITY_MIN_TTL_SECONDS = 300 _EXECUTION_CAPABILITY_MAX_TTL_SECONDS = 1_800 @@ -1136,6 +1150,24 @@ def _claim_from_stale_check_mode_row( "incident_semantic_route_rechecked": incident_routing_evaluated, } ) + for field_name in _STALE_CLAIM_REPLAY_CONTROL_FIELDS: + if field_name in input_payload: + claim_input[field_name] = input_payload[field_name] + row_tags = { + str(tag) + for tag in (row.get("tags") or []) + if isinstance(tag, str) + } + historical_shadow_replay = bool( + row_tags & _HISTORICAL_SHADOW_REPLAY_TAGS + ) + if ( + historical_shadow_replay + or input_payload.get("historical_projection_backfill") is True + or input_payload.get("telegram_provider_delivery") == "shadow_only" + ): + claim_input["historical_projection_backfill"] = True + claim_input["telegram_provider_delivery"] = "shadow_only" return AnsibleCheckModeClaim( op_id=str(row.get("op_id") or ""), source_candidate_op_id=source_candidate_op_id, @@ -7443,6 +7475,7 @@ async def claim_stale_pending_check_modes( SELECT check_mode.op_id, check_mode.parent_op_id, + check_mode.tags, coalesce( check_mode.incident_id::text, check_mode.input ->> 'incident_id' diff --git a/apps/api/tests/test_host_disk_pressure_ansible_catalog.py b/apps/api/tests/test_host_disk_pressure_ansible_catalog.py index 1314b40c0..52fb479e8 100644 --- a/apps/api/tests/test_host_disk_pressure_ansible_catalog.py +++ b/apps/api/tests/test_host_disk_pressure_ansible_catalog.py @@ -17,6 +17,7 @@ from src.services.awooop_ansible_check_mode_service import ( _send_controlled_apply_telegram_receipt, claim_catalog_drift_failed_check_modes, claim_semantically_misrouted_verified_applies, + claim_stale_pending_check_modes, run_pending_check_modes_once, ) from src.services.awooop_ansible_post_verifier import postconditions_for_catalog @@ -100,6 +101,7 @@ def test_stale_generic_host188_row_is_semantically_remapped() -> None: { "op_id": "00000000-0000-0000-0000-000000000188", "parent_op_id": "00000000-0000-0000-0000-000000000100", + "tags": ["ansible", "check_mode", "catalog_drift_replay"], "incident_id": "INC-DISK-188", "incident_alertname": "HostOutOfDiskSpace", "incident_alert_category": "host_resource", @@ -114,6 +116,15 @@ def test_stale_generic_host188_row_is_semantically_remapped() -> None: "playbook_path": "infra/ansible/playbooks/110-devops.yml", "inventory_hosts": ["host_110"], "risk_level": "medium", + "automation_run_id": ( + "00000000-0000-0000-0000-000000000188" + ), + "catalog_drift_replay": True, + "replay_of_check_mode_op_id": ( + "00000000-0000-0000-0000-000000000099" + ), + "historical_projection_backfill": True, + "telegram_provider_delivery": "shadow_only", }, } ) @@ -123,6 +134,37 @@ def test_stale_generic_host188_row_is_semantically_remapped() -> None: assert claim.inventory_hosts == ("host_188",) assert claim.input_payload["catalog_route_changed"] is True assert claim.input_payload["incident_semantic_route_rechecked"] is True + assert claim.input_payload["catalog_drift_replay"] is True + assert claim.input_payload["historical_projection_backfill"] is True + assert claim.input_payload["telegram_provider_delivery"] == "shadow_only" + + +def test_stale_catalog_replay_tag_recovers_missing_shadow_policy() -> None: + claim = _claim_from_stale_check_mode_row( + { + "op_id": "00000000-0000-0000-0000-000000000198", + "parent_op_id": "00000000-0000-0000-0000-000000000100", + "tags": ["ansible", "check_mode", "catalog_drift_replay"], + "incident_id": "INC-DISK-188-TAG-FALLBACK", + "incident_alertname": "HostOutOfDiskSpace", + "incident_alert_category": "host_resource", + "incident_affected_services": ["node-exporter-188"], + "input": { + "source_candidate_op_id": ( + "00000000-0000-0000-0000-000000000100" + ), + "incident_id": "INC-DISK-188-TAG-FALLBACK", + "catalog_id": "ansible:110-devops", + "playbook_path": "infra/ansible/playbooks/110-devops.yml", + "inventory_hosts": ["host_110"], + "risk_level": "medium", + }, + } + ) + + assert claim is not None + assert claim.input_payload["historical_projection_backfill"] is True + assert claim.input_payload["telegram_provider_delivery"] == "shadow_only" def test_disk_catalogs_have_independent_threshold_verifiers() -> None: @@ -141,12 +183,14 @@ def test_disk_catalogs_have_independent_threshold_verifiers() -> None: def test_historical_catalog_replay_is_per_failed_row_and_provider_silent() -> None: claim_source = inspect.getsource(claim_catalog_drift_failed_check_modes) + stale_source = inspect.getsource(claim_stale_pending_check_modes) send_source = inspect.getsource(_send_controlled_apply_telegram_receipt) assert "replay_of_check_mode_op_id" in claim_source assert "failed_check_mode_op_id" in claim_source assert '"historical_projection_backfill": True' in claim_source assert '"telegram_provider_delivery": "shadow_only"' in claim_source + assert "check_mode.tags" in stale_source assert "telegram_provider_delivery" in send_source From fa61d4e07ade204f8254925e64788a7cb65c694f Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 01:55:48 +0800 Subject: [PATCH 2/5] fix(agent99): dispatch durable alerts before LLM --- apps/api/src/api/v1/webhooks.py | 110 +++++++++++++--- .../tests/test_alertmanager_rule_bypass.py | 123 ++++++++++++++++++ 2 files changed, 215 insertions(+), 18 deletions(-) diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 3089895fe..6f4c66155 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -2135,7 +2135,6 @@ async def _process_new_alert_background( """ try: service = get_approval_service() - openclaw = get_openclaw() traced_alert_labels = { **(alert_labels or {}), "fingerprint": fingerprint, @@ -2147,8 +2146,19 @@ async def _process_new_alert_background( source_severity=severity, ) + agent99_durable_route = resolve_agent99_durable_route( + alertname=alertname, + severity=severity, + target_resource=target_resource, + namespace=namespace, + message=message, + labels=traced_alert_labels, + annotations=alert_context.get("annotations", {}), + ) rule_response = match_rule(alert_context) - should_bypass_llm = _should_use_alertmanager_rule_first(rule_response, alert_category) + should_bypass_llm = bool(agent99_durable_route) or ( + _should_use_alertmanager_rule_first(rule_response, alert_category) + ) if should_bypass_llm: logger.info( @@ -2157,7 +2167,12 @@ async def _process_new_alert_background( alertname=alertname, rule_id=rule_response.get("rule_id", ""), alert_category=alert_category, - reason="host/backup YAML 權威規則命中，跳過 LLM 避免產生錯誤 K8s 動作", + agent99_route=(agent99_durable_route or {}).get("route_id"), + reason=( + "agent99_durable_route_precedes_llm" + if agent99_durable_route + else "host_or_backup_authoritative_rule_precedes_llm" + ), ) risk_mapping = { "low": RiskLevel.LOW, @@ -2169,6 +2184,20 @@ async def _process_new_alert_background( str(rule_response.get("risk_level", "low")).lower(), RiskLevel.LOW, ) + if agent99_durable_route: + rule_risk = ( + RiskLevel.MEDIUM + if agent99_durable_route.get("suggested_mode") + in { + "Recover", + "StartVMs", + "HarborRepair", + "AwoooRepair", + "Perf", + "LoadShed", + } + else RiskLevel.LOW + ) blast = rule_response.get("blast_radius", {}) or {} impact_mapping = { "NONE": DataImpact.NONE, @@ -2180,6 +2209,12 @@ async def _process_new_alert_background( str(blast.get("data_impact", "NONE")).upper(), DataImpact.NONE, ) + if agent99_durable_route: + data_impact = ( + DataImpact.WRITE + if rule_risk == RiskLevel.MEDIUM + else DataImpact.READ_ONLY + ) rule_kubectl = str(rule_response.get("kubectl_command", "")).strip() rule_description = str(rule_response.get("description", message)) rule_action = ( @@ -2187,6 +2222,16 @@ async def _process_new_alert_background( if rule_kubectl else f"NO_ACTION - {rule_description[:120]}" ) + if agent99_durable_route: + rule_kubectl = "" + rule_description = ( + "告警已命中 Agent99 durable route；先建立同一 incident/run ledger，" + "再由 99 主機執行受控處置與獨立 verifier。" + ) + rule_action = ( + "AGENT99_CONTROLLED_ROUTE " + f"{agent99_durable_route['route_id']}" + ) _matched_playbook_id_cs2 = await resolve_playbook_id_for_alert( rule_id=str(rule_response.get("rule_id", "")), alertname=alertname, @@ -2214,6 +2259,34 @@ async def _process_new_alert_background( "playbook_id": _matched_playbook_id_cs2, **_guarded_action_cs2.metadata, } + if agent99_durable_route: + _approval_metadata_cs2.update({ + "source": "agent99_durable_route", + "agent99_route_id": agent99_durable_route["route_id"], + "agent99_kind": agent99_durable_route["kind"], + "agent99_mode": agent99_durable_route["suggested_mode"], + "llm_blocking_dependency": False, + }) + rule_dry_run_checks = [ + DryRunCheck( + name="規則命中", + passed=True, + message=str(rule_response.get("rule_id", "unknown")), + ), + DryRunCheck( + name="來源", + passed=True, + message="alertmanager", + ), + ] + if agent99_durable_route: + rule_dry_run_checks.append( + DryRunCheck( + name="Agent99 durable route", + passed=True, + message=agent99_durable_route["route_id"], + ) + ) approval_create = ApprovalRequestCreate( action=rule_action, description=f"[Rule: {rule_response.get('rule_id', 'unknown')}] {rule_description}", @@ -2229,19 +2302,12 @@ async def _process_new_alert_background( ), data_impact=data_impact, ), - dry_run_checks=[ - DryRunCheck( - name="規則命中", - passed=True, - message=str(rule_response.get("rule_id", "unknown")), - ), - DryRunCheck( - name="來源", - passed=True, - message="alertmanager", - ), - ], - requested_by="OpenClaw (rule-engine)", + dry_run_checks=rule_dry_run_checks, + requested_by=( + "Agent99 durable router" + if agent99_durable_route + else "OpenClaw (rule-engine)" + ), metadata=_approval_metadata_cs2, matched_playbook_id=_matched_playbook_id_cs2, ) @@ -2329,8 +2395,9 @@ async def _process_new_alert_background( ) _is_heartbeat = is_heartbeat_alertname(alertname) + _controlled_handoff: dict[str, Any] | None = None if _controlled_ai_policy_allows(rule_risk) and not _is_heartbeat: - await _try_auto_repair_background( + _controlled_handoff = await _try_auto_repair_background( incident_id=incident_id, approval_id=str(approval.id), alert_type=alert_type, @@ -2383,9 +2450,15 @@ async def _process_new_alert_background( estimated_downtime=str(blast.get("estimated_downtime", "N/A")), hit_count=1, primary_responsibility=str( - rule_response.get("primary_responsibility", "INFRA") + "AGENT99_CONTROLLED_EXECUTOR" + if agent99_durable_route + else rule_response.get("primary_responsibility", "INFRA") ), confidence=float(rule_response.get("confidence", 0.0) or 0.0), + automation_state=str( + (_controlled_handoff or {}).get("status") + or "agent99_dispatch_not_requested" + ), namespace=namespace, incident_id=incident_id, notification_type=notification_type, @@ -2397,6 +2470,7 @@ async def _process_new_alert_background( record_alert_chain_success("alertmanager") return + openclaw = get_openclaw() analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url, ai_tokens, ai_cost = await _analyze_alertmanager_with_timeout( openclaw, alert_context, diff --git a/apps/api/tests/test_alertmanager_rule_bypass.py b/apps/api/tests/test_alertmanager_rule_bypass.py index cc053e082..64f09135b 100644 --- a/apps/api/tests/test_alertmanager_rule_bypass.py +++ b/apps/api/tests/test_alertmanager_rule_bypass.py @@ -2,16 +2,20 @@ import ast import asyncio import inspect from datetime import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock import pytest from src.api.v1 import webhooks as webhooks_module from src.api.v1.webhooks import ( _analyze_alertmanager_with_timeout, + _process_new_alert_background, _should_bypass_alertmanager_llm, _should_use_alertmanager_rule_first, ) from src.repositories.alert_operation_log_repository import ALERT_EVENT_TYPES +from src.services.alert_approval_guard import ApprovalActionGuardResult from src.services.alertmanager_llm_guard import ( ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS, alertmanager_llm_inflight_key, @@ -175,6 +179,125 @@ async def test_alertmanager_analysis_error_returns_fallback(): assert result == (None, "fallback_error", "", None, "", 0, 0.0) +@pytest.mark.asyncio +async def test_agent99_durable_route_dispatches_before_llm(monkeypatch): + class ApprovalService: + created_request = None + + async def create_approval_with_fingerprint(self, *, request, fingerprint): + self.created_request = request + assert fingerprint == "fp-agent99-durable-route" + return SimpleNamespace(id="approval-agent99-fast-path", incident_id=None) + + async def update_incident_id(self, approval_id, incident_id): + assert str(approval_id) == "approval-agent99-fast-path" + assert incident_id == "INC-AGENT99-FAST-PATH" + + service = ApprovalService() + openclaw_factory = Mock( + side_effect=AssertionError("durable Agent99 route must not create OpenClaw") + ) + handoff = AsyncMock( + return_value={ + "status": "agent99_dispatch_accepted_verifier_pending", + "queued": True, + } + ) + telegram = AsyncMock() + + monkeypatch.setattr(webhooks_module, "get_approval_service", lambda: service) + monkeypatch.setattr(webhooks_module, "get_openclaw", openclaw_factory) + monkeypatch.setattr( + webhooks_module, + "get_trusted_alert_canonical_route_context", + lambda *_args, **_kwargs: {}, + ) + monkeypatch.setattr( + webhooks_module, + "match_rule", + lambda _context: { + "rule_id": "generic_fallback", + "risk_level": "critical", + "blast_radius": {}, + "kubectl_command": "", + "description": "generic fallback must not block Agent99", + "confidence": 0.0, + }, + ) + monkeypatch.setattr( + webhooks_module, + "resolve_playbook_id_for_alert", + AsyncMock(return_value=None), + ) + monkeypatch.setattr( + webhooks_module, + "guard_alert_approval_action", + AsyncMock( + side_effect=lambda **kwargs: ApprovalActionGuardResult( + action=kwargs["action"] + ) + ), + ) + monkeypatch.setattr( + webhooks_module, + "get_auto_approve_policy", + lambda: SimpleNamespace( + evaluate=lambda _proposal: SimpleNamespace( + should_auto_approve=False, + reason=SimpleNamespace(value="bounded_durable_route"), + ) + ), + ) + monkeypatch.setattr( + webhooks_module, + "create_incident_for_approval", + AsyncMock(return_value="INC-AGENT99-FAST-PATH"), + ) + monkeypatch.setattr( + webhooks_module, + "record_alertmanager_event", + AsyncMock(), + ) + monkeypatch.setattr(webhooks_module, "_try_auto_repair_background", handoff) + monkeypatch.setattr(webhooks_module, "_push_to_telegram_background", telegram) + monkeypatch.setattr(webhooks_module, "record_alert_chain_success", Mock()) + + await _process_new_alert_background( + alert_context={ + "alertname": "RebootAutoRecoveryActiveBlocker", + "annotations": {"summary": "cold-start gate blocked"}, + "source_url": "http://prometheus.invalid/graph", + }, + alert_id="alert-agent99-fast-path", + fingerprint="fp-agent99-durable-route", + target_resource="reboot-auto-recovery-slo", + namespace="default", + alert_type="custom", + message="cold-start-gate service recovery verifier", + alertname="RebootAutoRecoveryActiveBlocker", + severity="critical", + alert_labels={"service": "cold-start-gate"}, + notification_type="TYPE-3", + alert_category="general", + can_auto_repair=True, + ) + + openclaw_factory.assert_not_called() + handoff.assert_awaited_once() + assert handoff.await_args.kwargs["source_alertname"] == ( + "RebootAutoRecoveryActiveBlocker" + ) + assert service.created_request.risk_level.value == "medium" + assert service.created_request.action == ( + "AGENT99_CONTROLLED_ROUTE agent99:host_recovery:Recover" + ) + assert service.created_request.metadata["source"] == "agent99_durable_route" + telegram.assert_awaited_once() + assert telegram.await_args.kwargs["automation_state"] == ( + "agent99_dispatch_accepted_verifier_pending" + ) + + def test_resolved_guard_stamp_without_timestamp_is_clean(): assert _format_resolved_guard_stamp(None) == "✅ 此事件已解決" From a31a278e585d9a186d2110495ea856df98504bfa Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 02:05:14 +0800 Subject: [PATCH 3/5] fix(recovery): align host112 cold-start metric scope --- .../services/agent99_same_run_reconcile.py | 22 +++---- .../tests/test_agent99_same_run_reconcile.py | 11 ++++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 14 +++-- docs/runbooks/REBOOT-RECOVERY-SOP.md | 12 +++- .../roles/cold-start-monitor/tasks/main.yml | 2 +- ops/monitoring/alerts-unified.yml | 24 ++++---- .../full-stack-cold-start-baseline.yml | 2 +- .../ops/recovery-scorecard-contract-check.py | 2 +- .../full-stack-recovery-scorecard.sh | 6 +- .../test_cold_start_monitor_bounded_probes.py | 17 ++++++ .../verify-cold-start-monitor-deploy.sh | 58 +++++++++++++++++++ 11 files changed, 134 insertions(+), 36 deletions(-) diff --git a/apps/api/src/services/agent99_same_run_reconcile.py b/apps/api/src/services/agent99_same_run_reconcile.py index ed4e10555..bc16244fc 100644 --- a/apps/api/src/services/agent99_same_run_reconcile.py +++ b/apps/api/src/services/agent99_same_run_reconcile.py @@ -38,41 +38,41 @@ AGENT99_SAME_RUN_STATUS_VERIFIER = ( "agent99-status-same-run-no-write-post-condition-v1" ) AGENT99_COLD_START_METRIC_HOST = "110" -AGENT99_COLD_START_METRIC_SCOPE = "110_120_121_188" +AGENT99_COLD_START_METRIC_SCOPE = "110_112_120_121_188" AGENT99_COLD_START_MAX_AGE_SECONDS = 900 _COLD_START_QUERIES = { "monitor_up": ( - 'awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_monitor_up{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'mode="read_only"}' ), "pass_gates": ( - 'awoooi_cold_start_pass_gates{host="110",scope="110_120_121_188"}' + f'awoooi_cold_start_pass_gates{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}"}}' ), "warn_gates": ( - 'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}' + f'awoooi_cold_start_warn_gates{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}"}}' ), "blocked_gates": ( - 'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}' + f'awoooi_cold_start_blocked_gates{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}"}}' ), "last_run_timestamp": ( - 'awoooi_cold_start_last_run_timestamp{host="110",' - 'scope="110_120_121_188"}' + f'awoooi_cold_start_last_run_timestamp{{host="110",' + f'scope="{AGENT99_COLD_START_METRIC_SCOPE}"}}' ), "green_result": ( - 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_last_result{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'result="green"}' ), "degraded_result": ( - 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_last_result{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'result="degraded"}' ), "blocked_result": ( - 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_last_result{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'result="blocked"}' ), "check_failed_result": ( - 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_last_result{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'result="check_failed"}' ), "firing_blocking_alerts": ( diff --git a/apps/api/tests/test_agent99_same_run_reconcile.py b/apps/api/tests/test_agent99_same_run_reconcile.py index 78f92f100..4fae7764b 100644 --- a/apps/api/tests/test_agent99_same_run_reconcile.py +++ b/apps/api/tests/test_agent99_same_run_reconcile.py @@ -117,6 +117,17 @@ def _evidence_refs() -> dict[str, str]: } +def test_cold_start_source_queries_include_host112_scope() -> None: + expected_scope = 'scope="110_112_120_121_188"' + + assert reconcile.AGENT99_COLD_START_METRIC_SCOPE == "110_112_120_121_188" + for name, query in reconcile._COLD_START_QUERIES.items(): + if name == "firing_blocking_alerts": + continue + assert expected_scope in query + assert 'scope="110_120_121_188"' not in query + + def test_cold_start_source_resolution_allows_warning_only_but_requires_fresh_zero( monkeypatch, ) -> None: diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 647a0d9b7..162aca170 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.136 -> Last updated: 2026-07-14 Asia/Taipei +> Version: v1.137 +> Last updated: 2026-07-15 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 --- @@ -10,6 +10,12 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天，必須先重跑 live check，再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 +### v1.137 2026-07-15 Cold-start metric scope 全鏈路防漂移 + +Cold-start exporter 的 canonical scope 固定為 `110_112_120_121_188`。producer、`ops/monitoring/alerts-unified.yml` 的 recording/alert rules、Agent99 same-run reconciler、full-stack recovery scorecard、baseline 與 Ansible post-verifier 必須使用同一個 label value；任一 consumer 仍查詢舊 `110_120_121_188` 時，會把正常的新 series 誤判成缺失並產生假的 `ColdStartMonitorMissing`，因此只能標示 monitoring contract drift，不能觸發主機或服務重啟。 + +每次 exporter scope 或 required host inventory 變更後，`verify-cold-start-monitor-deploy.sh` 除了 script hash、textfile 與 host112 blocker series，也必須讀 Prometheus `/api/v1/rules` 驗證所有 cold-start recording/alert expressions已切到 canonical scope、舊 scope 為零，並確認 reload 後 `ColdStartMonitorMissing` 不再 firing。禁止人工 silence 或刪除告警掩蓋 drift；先修 canonical source、走受控 rules deploy/reload，再以 Prometheus rule readback、recovery scorecard 與 Agent99 same-run source verifier 共同收斂。 + ### v1.136 2026-07-14 Host112 正式納入 cold-start P0 release gate `full-stack-cold-start-check.sh` 舊版仍顯示 `112 Kali is intentionally skipped`，與 Windows99 五台 required VM、Agent99 `host112_guest_readiness` phase 及全主機 10 分鐘 SLO 不一致。現行 baseline 已把 112 從 excluded host 移入 included host；P0 network 固定檢查 ping / TCP 22，並透過 `kali@192.168.0.112` 執行 canonical `/usr/local/sbin/awoooi-host112-guest-readiness --check`。release gate 必須同時驗證 readback schema、systemd running、guest/console/services/recovery timer、Wazuh 1514/1515/55000 independent verifier 與 `runtime_write_performed=0`。任何一項失敗均維持 BLOCKED；不得因其他四台 Linux host 或 public routes 為 200 而宣稱全主機恢復。110 textfile exporter scope 固定為 `110_112_120_121_188`，並分開輸出 112 host unreachable 與 guest readiness failed blocker series；`verify-cold-start-monitor-deploy.sh` 必須驗證 source/live hash、scope 與 blocker series parity，避免 checker 已納管但 Prometheus 仍漏掉 112。 @@ -67,7 +73,7 @@ v1.80 / v1.81 credential escrow intake scorecard rule：同一輪 owner response v1.93 reboot SLA readback fixed fields rule：`scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py`、`docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json` 與 `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須固定輸出 `current_phase`、`eta_or_wait_reason`、`primary_blocker`、`active_blockers`、`next_safe_action` 與 `reboot_sop_progress.fixed_triage_order`。回報時先讀這些欄位，不得重新臨場發明排查順序；若 `current_phase=host_boot_detection_blocked` 且 `eta_or_wait_reason=target_window_elapsed_eta_unavailable_until_fresh_all_host_reboot_event_and_probe`，只能宣稱「10 分鐘 SLO 尚未證明」，下一步固定補 fresh all-host reboot event / probe 與 99 VMware/Windows readback，不得把部分 Linux host green 當全主機 green。 -v1.94 Windows 99 / VMware verifier rule：99 主機與 guest VM 自動啟動不得再只靠「ping 到 99」或「VMX source 存在」宣稱。`scripts/reboot-recovery/windows99-vmware-autostart.ps1 -Mode Verify` 必須輸出 no-secret 固定欄位：`VMRUN_PRESENT`、`VMX alias=... present=...`、`VMWARE_SERVICE ... ok=...`、`VMWARE_AUTOSTART_TASK ... ok=...`、`WINDOWS_UPDATE_POLICY ... ok=...`、`VM_POWER alias=... running=...`、`VMWARE_AUTOSTART_CONFIG_READY`、`VMWARE_AUTOSTART_POWER_READY`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`、`VMWARE_AUTOSTART_VERIFY_READY`。`reboot-auto-recovery-slo-scorecard.py --windows99-vmware-file ` 必須解析這些欄位；缺 readback 時 active blocker 固定為 `windows99_vmware_autostart_readback_missing`，next action 固定為 `collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot`。預設 required guest VM alias 以本輪 reboot 事故 source-of-truth 為準：`111 / 188 / 120 / 121 / 112`；`110` 保留為 required host / control-plane reachability，不得被當成 99 VMware 預設 VM。此 verifier 不讀 Windows 密碼、不讀 secret、不啟動 VM、不重啟 host；`Apply` 仍需獨立 controlled apply 與 post-verifier。 +v1.94 Windows 99 / VMware verifier rule：99 主機與 guest VM 自動啟動不得再只靠「ping 到 99」或「VMX source 存在」宣稱。`scripts/reboot-recovery/windows99-vmware-autostart.ps1 -Mode Verify` 必須輸出 no-secret 固定欄位：`VMRUN_PRESENT`、`VMX alias=... present=...`、`VMWARE_SERVICE ... ok=...`、`VMWARE_AUTOSTART_TASK ... ok=...`、`WINDOWS_UPDATE_POLICY ... ok=...`、`VM_POWER alias=... running=...`、`VMWARE_AUTOSTART_CONFIG_READY`、`VMWARE_AUTOSTART_POWER_READY`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`、`VMWARE_AUTOSTART_VERIFY_READY`。`reboot-auto-recovery-slo-scorecard.py --windows99-vmware-file ` 必須解析這些欄位；缺 readback 時 active blocker 固定為 `windows99_vmware_autostart_readback_missing`，next action 固定為 `collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot`。現行 required guest VM alias 依 live inventory 固定為 `110 / 188 / 120 / 121 / 112`；`111` 是獨立 Darwin 主機，不得被當成 Windows99 VM。舊 `111 / 188 / 120 / 121 / 112` alias contract 已由 `global_product_governance_v2`、v1.133 與本規則取代。此 verifier 不讀 Windows 密碼、不讀 secret、不啟動 VM、不重啟 host；`Apply` 仍需獨立 controlled apply 與 post-verifier。 v1.95 Windows 99 management-channel readback rule：若 99 可 ping / RDP / TCP，但 `windows99-vmware-autostart.ps1 -Mode Verify` 尚未收回，不得只寫成「等 verifier」。必須先跑 `scripts/reboot-recovery/windows99-management-channel-probe.py --output `，並把結果用 `reboot-auto-recovery-slo-scorecard.py --windows99-management-file ` 接入同一份 SLO scorecard。此 probe 只做 no-secret readback：TCP `22 / 135 / 445 / 2179 / 3389 / 5985 / 5986`、SSH BatchMode publickey、WinRM port、RDP console reachability 與 VMConnect / local console reachability；不得讀 Windows 密碼、不得啟動 VM、不得重啟、不得改 Windows Update。2026-07-02 18:28 live readback：`host_reachable=true`、`rdp_console_reachable=true`、`hyperv_vmconnect_open=true`、`local_console_channel_reachable=true`、`console_collection_channels=["rdp_console","hyperv_vmconnect"]`、`ssh_batch.status=permission_denied`、`winrm_http_open=false`、`winrm_https_open=false`、`remote_execution_channel_ready=false`；因此 active blockers 必須包含 `windows99_remote_execution_channel_unavailable` 與 `windows99_vmware_autostart_readback_missing`，next action 固定為 `restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot`。 @@ -1426,7 +1432,7 @@ docker exec momo-pro-system python -c 'from services.cache_service import clear_ ## 14. 主機開機、關機、重啟 SOP -本節是每次 110 / 120 / 121 / 188 相關電源操作的標準程序。112 是 Kali，只保留 read-only evidence，不納入本輪恢復或例行重啟釋出。 +本節是每次 110 / 112 / 120 / 121 / 188 相關電源操作的標準程序。112 是 Windows99 上的 required Kali guest，必須通過 VM power、boot identity、SSH forced-command、systemd、console、Wazuh 與 recovery timer readback 才能釋出；舊版「112 不納入恢復」已由 `global_product_governance_v2` 與 v1.136 取代。 ### 14.1 共同紅線 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 6d6065ac6..e80956cc1 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.32 -> **最後更新**: 2026-07-14 (台北時間) +> **版本**: v5.33 +> **最後更新**: 2026-07-15 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-14 Agent99 host111/host112 readback false-alert closure +> **觸發事件**: 2026-07-15 cold-start metric scope / Agent99 same-run false-alert closure --- @@ -21,6 +21,12 @@ ## 架構概覽與依賴圖 +### 2026-07-15 Cold-start scope 全鏈路防漂移 + +Cold-start exporter、Prometheus recording/alert rules、Agent99 same-run reconciler、recovery scorecard、baseline 與 Ansible verifier 的 canonical metric scope 一律是 `110_112_120_121_188`。若 producer 已輸出此 scope，但任何 consumer 仍查詢舊 `110_120_121_188`，Prometheus 會產生假的 `ColdStartMonitorMissing`，Agent99 callback 也會因 source verifier 缺資料而 fail closed。這是監控合約漂移，不是主機離線；禁止以 silence、重啟主機或重啟產品服務處理。 + +受控修復順序固定為：source contract diff -> focused tests / rule parse -> Gitea CD/deploy marker -> canonical rules deploy/reload -> Prometheus `/api/v1/rules` expression readback -> firing alert resolution -> recovery scorecard -> Agent99 same-run callback verifier。`verify-cold-start-monitor-deploy.sh` 必須驗證 active rules 全部使用 canonical scope 且舊 scope 為零；缺任一 receipt 只能標示 partial/degraded，不能宣稱 Agent99 自動修復閉環完成。 + ### 2026-07-14 Agent99 host readback false-alert closure 111 是獨立 Darwin 主機，不是 Windows99 VM。Agent99 必須以 `ooo@192.168.0.111` 經 110 的 restricted forced-command route 讀取狀態；使用預設 `wooo` 會得到 SSH failure，不能包裝成 CPU 壓力。forced command 固定輸出 boot/Ollama 加 `agent99_perf_readback_v1` 四項唯讀指標，仍禁止 shell、PTY、forwarding、secret read 與任意命令。Windows99 deploy/bootstrap 必須自動遷移並 post-verify `sshUsers["192.168.0.111"]="ooo"`；Perf 只有在 cores/load/memory/disk 全部可解析時才可通過。 diff --git a/infra/ansible/roles/cold-start-monitor/tasks/main.yml b/infra/ansible/roles/cold-start-monitor/tasks/main.yml index 798250287..0cf34e28c 100644 --- a/infra/ansible/roles/cold-start-monitor/tasks/main.yml +++ b/infra/ansible/roles/cold-start-monitor/tasks/main.yml @@ -66,7 +66,7 @@ - name: "cold-start monitor | 驗證 green metric 存在" ansible.builtin.command: cmd: >- - grep -q 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} 1' + grep -q 'awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"} 1' {{ cold_start_monitor_textfile_dir }}/cold_start_recovery.prom become: true become_user: "{{ cold_start_monitor_user }}" diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 253873d5f..f30809f58 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -24,16 +24,16 @@ groups: - record: awoooi_recovery_core_ready expr: | sum without(result) ( - awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} == bool 1 + awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"} == bool 1 ) * on(host,scope) ( - awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"} == bool 0 + awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"} == bool 0 ) * on(host,scope) ( - awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"} == bool 0 + awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"} == bool 0 ) * on(host,scope) ( - (time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_120_121_188"}) < bool 3600 + (time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_112_120_121_188"}) < bool 3600 ) - record: awoooi_recovery_dr_offsite_ready @@ -2326,7 +2326,7 @@ groups: runbook: "檢查 `/home/wooo/logs/prometheus-rule-drift-guard.log` 與 `/home/wooo/monitoring/alerts.yml.guard.bak.*`，找出誰覆寫了 active rules。" - alert: ColdStartMonitorMissing - expr: absent(awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188"}) + expr: absent(awoooi_cold_start_monitor_up{host="110",scope="110_112_120_121_188"}) for: 15m labels: severity: warning @@ -2342,7 +2342,7 @@ groups: runbook: "執行 scripts/reboot-recovery/install-cold-start-monitor-110.sh，確認 /home/wooo/node_exporter_textfiles/cold_start_recovery.prom" - alert: ColdStartMonitorStale - expr: time() - awoooi_cold_start_last_run_timestamp{host="110",scope="110_120_121_188"} > 900 + expr: time() - awoooi_cold_start_last_run_timestamp{host="110",scope="110_112_120_121_188"} > 900 for: 10m labels: severity: warning @@ -2358,7 +2358,7 @@ groups: runbook: "SSH 110 檢查 crontab、/tmp/awoooi-cold-start-monitor.cron.log、cold-start-last.log" - alert: ColdStartRecoveryBlocked - expr: awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"} > 0 + expr: awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"} > 0 for: 5m labels: severity: critical @@ -2374,7 +2374,7 @@ groups: runbook: "查看 /home/wooo/reboot-recovery/cold-start-last.log；依 docs/runbooks/FULL-STACK-COLD-START-SOP.md 的 P0→P2 順序修復" - alert: K3sNodeFilesystemErrorGateBlocked - expr: awoooi_cold_start_blocker_reason{host="110",scope="110_120_121_188",reason="k3s_node_filesystem_error",target="120"} > 0 + expr: awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="k3s_node_filesystem_error",target="120"} > 0 for: 5m labels: severity: critical @@ -2392,7 +2392,7 @@ groups: runbook: "查看 110 `/home/wooo/reboot-recovery/cold-start-last.log`，執行 `scripts/reboot-recovery/120-fsck-maintenance-checklist.sh` 做維護前只讀檢查；維護窗內用 console/rescue 對 120 root LV 執行 fsck，禁止 online fsck。" - alert: ColdStartHost120Unreachable - expr: awoooi_cold_start_blocker_reason{host="110",scope="110_120_121_188",reason="host_unreachable",target="120"} > 0 + expr: awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="host_unreachable",target="120"} > 0 for: 3m labels: severity: critical @@ -2410,7 +2410,7 @@ groups: runbook: "查看 120 console。若停在 initramfs/manual fsck，先對 root LV 做離線 fsck；若主機關機或網卡異常，先恢復電源/網路，再重跑 full-stack cold-start gate。禁止從自動修復直接重啟其他服務掩蓋主機離線。" - alert: ColdStartRecoveryDegraded - expr: awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"} > 0 + expr: awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"} > 0 for: 15m labels: severity: warning @@ -2426,7 +2426,7 @@ groups: runbook: "查看 /home/wooo/reboot-recovery/cold-start-last.log，修到 PASS/WARN/BLOCKED = green" - alert: ColdStartLastGreenTooOld - expr: time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_120_121_188"} > 3600 + expr: time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_112_120_121_188"} > 3600 for: 15m labels: severity: warning @@ -2438,7 +2438,7 @@ groups: auto_repair: "false" annotations: summary: "Full-stack cold-start gate has not been GREEN recently" - description: "距離上次 GREEN 已超過 {{ $value | humanizeDuration }}；需要確認 110/120/121/188 與排程/網站 gate。" + description: "距離上次 GREEN 已超過 {{ $value | humanizeDuration }}；需要確認 110/112/120/121/188 與排程/網站 gate。" runbook: "執行 SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --send-alert-test" # ========================================================================= diff --git a/ops/reboot-recovery/full-stack-cold-start-baseline.yml b/ops/reboot-recovery/full-stack-cold-start-baseline.yml index d0782f042..e1bde6fa3 100644 --- a/ops/reboot-recovery/full-stack-cold-start-baseline.yml +++ b/ops/reboot-recovery/full-stack-cold-start-baseline.yml @@ -351,7 +351,7 @@ authoritative_checks: expected: PASS=64 WARN=0 BLOCKED=0 textfile_metric: path: /home/wooo/node_exporter_textfiles/cold_start_recovery.prom - green_metric: awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} 1 + green_metric: awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"} 1 backup_baseline: path: ops/reboot-recovery/full-stack-backup-baseline.yml required_metrics: diff --git a/scripts/ops/recovery-scorecard-contract-check.py b/scripts/ops/recovery-scorecard-contract-check.py index 5be1f88c1..7a9c98f6f 100755 --- a/scripts/ops/recovery-scorecard-contract-check.py +++ b/scripts/ops/recovery-scorecard-contract-check.py @@ -23,7 +23,7 @@ else: DEFAULT_RULES = Path("ops/monitoring/alerts-unified.yml") DEFAULT_BASELINE = Path("ops/reboot-recovery/full-stack-backup-baseline.yml") -EXPECTED_CORE = 'awoooi_recovery_core_ready{host="110",scope="110_120_121_188"}' +EXPECTED_CORE = 'awoooi_recovery_core_ready{host="110",scope="110_112_120_121_188"}' EXPECTED_DR = 'awoooi_recovery_dr_offsite_ready{host="110"}' diff --git a/scripts/reboot-recovery/full-stack-recovery-scorecard.sh b/scripts/reboot-recovery/full-stack-recovery-scorecard.sh index 223548373..27937e255 100755 --- a/scripts/reboot-recovery/full-stack-recovery-scorecard.sh +++ b/scripts/reboot-recovery/full-stack-recovery-scorecard.sh @@ -151,9 +151,9 @@ echo "AWOOOI full-stack recovery scorecard" date '+%Y-%m-%d %H:%M:%S %Z' echo -cold_green="$(bool_metric 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"}')" -cold_warn="$(metric_value 'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)" -cold_blocked="$(metric_value 'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)" +cold_green="$(bool_metric 'awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"}')" +cold_warn="$(metric_value 'awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"}' 2>/dev/null || echo 999)" +cold_blocked="$(metric_value 'awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"}' 2>/dev/null || echo 999)" cold_alerts="$(metric_value 'count(ALERTS{alertname=~"ColdStart.*",alertstate="firing"})' 2>/dev/null || echo 999)" status_value CORE_COLD_START_GREEN "$cold_green" diff --git a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py index 05270aaa4..7e06ecab2 100644 --- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py +++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py @@ -72,6 +72,14 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None: encoding="utf-8" ) deploy_verifier = VERIFY_DEPLOY.read_text(encoding="utf-8") + alerts = (ROOT / "ops" / "monitoring" / "alerts-unified.yml").read_text( + encoding="utf-8" + ) + recovery_scorecard = RECOVERY_SCORECARD.read_text(encoding="utf-8") + ansible_verifier = ( + ROOT / "infra" / "ansible" / "roles" / "cold-start-monitor" + / "tasks" / "main.yml" + ).read_text(encoding="utf-8") assert "Scope: 110 / 112 / 120 / 121 / 188." in text assert "for host in 110 112 120 121 188; do" in text @@ -86,6 +94,15 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None: assert 'reason="host_unreachable",target="112"' in exporter assert 'reason="guest_readiness_failed",target="112"' in exporter assert 'scope="110_112_120_121_188"' in deploy_verifier + assert 'base_url + "/api/v1/rules"' in deploy_verifier + assert "COLD_START_RULE_SCOPE_OK" in deploy_verifier + assert 'stale_scope = \'scope="110_120_121_188"\'' in deploy_verifier + assert 'scope="110_112_120_121_188"' in alerts + assert 'scope="110_112_120_121_188"' in recovery_scorecard + assert 'scope="110_112_120_121_188"' in ansible_verifier + assert 'scope="110_120_121_188"' not in alerts + assert 'scope="110_120_121_188"' not in recovery_scorecard + assert 'scope="110_120_121_188"' not in ansible_verifier def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None: diff --git a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh index c968df054..231d4dc9a 100755 --- a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh +++ b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh @@ -95,6 +95,63 @@ else: PY } +verify_cold_start_rule_scope() { + PROMETHEUS_URL="$PROMETHEUS_URL" python3 - <<'PY' +import json +import os +import urllib.request + +base_url = os.environ["PROMETHEUS_URL"].rstrip("/") +canonical_scope = 'scope="110_112_120_121_188"' +stale_scope = 'scope="110_120_121_188"' +required_rules = { + "awoooi_recovery_core_ready", + "ColdStartMonitorMissing", + "ColdStartMonitorStale", + "ColdStartRecoveryBlocked", + "K3sNodeFilesystemErrorGateBlocked", + "ColdStartHost120Unreachable", + "ColdStartRecoveryDegraded", + "ColdStartLastGreenTooOld", +} + +try: + payload = json.load( + urllib.request.urlopen(base_url + "/api/v1/rules", timeout=8) + ) +except Exception as exc: # noqa: BLE001 + raise SystemExit(f"BLOCKED cold-start rule scope readback failed: {exc}") from exc + +if payload.get("status") != "success": + raise SystemExit("BLOCKED cold-start rule scope readback returned non-success") + +queries = {} +for group in payload.get("data", {}).get("groups", []): + for rule in group.get("rules", []): + name = str(rule.get("name") or "") + if name in required_rules: + queries[name] = str(rule.get("query") or rule.get("expr") or "") + +missing = sorted(required_rules - queries.keys()) +wrong_scope = sorted( + name for name, query in queries.items() if canonical_scope not in query +) +stale = sorted(name for name, query in queries.items() if stale_scope in query) +if missing or wrong_scope or stale: + raise SystemExit( + "BLOCKED cold-start rule scope drift " + f"missing={','.join(missing) or 'none'} " + f"wrong_scope={','.join(wrong_scope) or 'none'} " + f"stale_scope={','.join(stale) or 'none'}" + ) + +print( + "COLD_START_RULE_SCOPE_OK " + f"scope=110_112_120_121_188 rules={len(required_rules)} stale_scope=0" +) +PY +} + report_runtime_state() { remote_read "awk ' /awoooi_cold_start_monitor_up/ {monitor_up=\$NF} @@ -139,6 +196,7 @@ require_remote_pattern \ "110 cold-start monitor exports host112 readiness blocker series" report_runtime_state +verify_cold_start_rule_scope report_cold_start_alerts echo "COLD_START_MONITOR_DEPLOY_PARITY_OK" From 74fbefc6c9587f5fe75ff5f7595472c1709c26be Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 02:09:54 +0800 Subject: [PATCH 4/5] fix(monitoring): make alert deploy fail closed --- scripts/ops/deploy-alerts.sh | 82 +++++++++++++++++-- .../test_cold_start_monitor_bounded_probes.py | 15 ++++ 2 files changed, 89 insertions(+), 8 deletions(-) diff --git a/scripts/ops/deploy-alerts.sh b/scripts/ops/deploy-alerts.sh index ce8eaf0a9..728edc91c 100755 --- a/scripts/ops/deploy-alerts.sh +++ b/scripts/ops/deploy-alerts.sh @@ -4,7 +4,7 @@ # 2026-04-05 Claude Code: Sprint 1 自動化部署 # 用法: bash scripts/ops/deploy-alerts.sh [--dry-run] -set -eo pipefail +set -euo pipefail ALERT_RULES_FILE="ops/monitoring/alerts-unified.yml" SLO_RULES_FILE="ops/monitoring/slo-rules.yml" @@ -15,10 +15,59 @@ TARGET_ALERTS_CANONICAL_PATH="/home/wooo/monitoring/alerts-unified.canonical.yml TARGET_SLO_PATH="/home/wooo/monitoring/slo-rules.yml" TARGET_DRIFT_GUARD_PATH="/home/wooo/scripts/prometheus-rule-drift-guard.sh" PROMETHEUS_URL="http://${TARGET_HOST}:9090" +PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}" +PROMETHEUS_RUNTIME_RULES="${PROMETHEUS_RUNTIME_RULES:-/etc/prometheus/alerts.yml}" DRY_RUN="${1:-}" +DEPLOY_STAMP="$(date +%Y%m%d%H%M%S)" +REMOTE_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml" +CONTAINER_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml" +TARGET_ALERTS_BACKUP_PATH="${TARGET_ALERTS_PATH}.bak.${DEPLOY_STAMP}" +TARGET_ALERTS_CANONICAL_BACKUP_PATH="${TARGET_ALERTS_CANONICAL_PATH}.bak.${DEPLOY_STAMP}" +TARGET_SLO_BACKUP_PATH="${TARGET_SLO_PATH}.bak.${DEPLOY_STAMP}" +TARGET_DRIFT_GUARD_BACKUP_PATH="${TARGET_DRIFT_GUARD_PATH}.bak.${DEPLOY_STAMP}" +APPLY_STARTED=0 +DEPLOY_VERIFIED=0 log() { echo "[$(date '+%H:%M:%S')] $*"; } +cleanup_candidate() { + ssh wooo@${TARGET_HOST} "rm -f '${REMOTE_CANDIDATE_PATH}'; docker exec -u 0 '${PROMETHEUS_CONTAINER}' rm -f '${CONTAINER_CANDIDATE_PATH}' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true +} + +rollback_deploy() { + log "部署驗證失敗，開始還原 ${DEPLOY_STAMP} 前規則" + ssh wooo@${TARGET_HOST} "set -euo pipefail + cp '${TARGET_ALERTS_BACKUP_PATH}' '${TARGET_ALERTS_PATH}' + cp '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}' '${TARGET_ALERTS_CANONICAL_PATH}' + cp '${TARGET_SLO_BACKUP_PATH}' '${TARGET_SLO_PATH}' + cp '${TARGET_DRIFT_GUARD_BACKUP_PATH}' '${TARGET_DRIFT_GUARD_PATH}' + chmod 0755 '${TARGET_DRIFT_GUARD_PATH}' + curl -fsS -X POST '${PROMETHEUS_URL}/-/reload' >/dev/null + sleep 2 + curl -fsS '${PROMETHEUS_URL}/-/ready' >/dev/null" + log "已還原規則並完成 Prometheus readiness readback" +} + +on_exit() { + local rc=$? + cleanup_candidate + if [ "$rc" -ne 0 ] && [ "$APPLY_STARTED" -eq 1 ] && [ "$DEPLOY_VERIFIED" -eq 0 ]; then + rollback_deploy || log "ERROR: 自動 rollback 失敗，保留精確 backup paths 供獨立恢復" + fi + exit "$rc" +} + +remote_promtool_preflight() { + scp "$ALERT_RULES_FILE" "wooo@${TARGET_HOST}:${REMOTE_CANDIDATE_PATH}" + ssh wooo@${TARGET_HOST} "set -euo pipefail + docker cp '${REMOTE_CANDIDATE_PATH}' '${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE_PATH}' + docker exec '${PROMETHEUS_CONTAINER}' promtool check rules '${CONTAINER_CANDIDATE_PATH}'" + cleanup_candidate + log "✅ remote promtool preflight 通過" +} + +trap on_exit EXIT + file_sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}' @@ -59,6 +108,7 @@ log "✅ YAML 語法驗證通過" # Dry run 模式 if [ "$DRY_RUN" = "--dry-run" ]; then + remote_promtool_preflight log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_PATH}" log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_CANONICAL_PATH}" log "DRY RUN: would deploy $SLO_RULES_FILE to ${TARGET_HOST}:${TARGET_SLO_PATH}" @@ -70,13 +120,17 @@ if [ "$DRY_RUN" = "--dry-run" ]; then exit 0 fi -# 備份現有規則 -ssh wooo@${TARGET_HOST} "\ - cp ${TARGET_ALERTS_PATH} ${TARGET_ALERTS_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \ - cp ${TARGET_ALERTS_CANONICAL_PATH} ${TARGET_ALERTS_CANONICAL_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \ - cp ${TARGET_SLO_PATH} ${TARGET_SLO_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \ - cp ${TARGET_DRIFT_GUARD_PATH} ${TARGET_DRIFT_GUARD_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true" +# 使用 production Prometheus 的 promtool 驗證候選規則，通過後才能寫 active path。 +remote_promtool_preflight + +# 備份現有規則；任一 source 缺失時 fail closed，不開始 apply。 +ssh wooo@${TARGET_HOST} "set -euo pipefail + cp '${TARGET_ALERTS_PATH}' '${TARGET_ALERTS_BACKUP_PATH}' + cp '${TARGET_ALERTS_CANONICAL_PATH}' '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}' + cp '${TARGET_SLO_PATH}' '${TARGET_SLO_BACKUP_PATH}' + cp '${TARGET_DRIFT_GUARD_PATH}' '${TARGET_DRIFT_GUARD_BACKUP_PATH}'" log "✅ 現有規則已備份" +APPLY_STARTED=1 # 部署新規則 scp "$ALERT_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_ALERTS_PATH} @@ -112,7 +166,7 @@ fi log "✅ 遠端規則 hash 驗證通過" # Reload Prometheus -ssh wooo@${TARGET_HOST} "curl -s -X POST ${PROMETHEUS_URL}/-/reload" +ssh wooo@${TARGET_HOST} "curl -fsS -X POST ${PROMETHEUS_URL}/-/reload >/dev/null" sleep 3 # 驗證規則數量 @@ -161,4 +215,16 @@ for rule in "${KEY_SLO_RULES[@]}"; do fi done +REMOTE_RUNTIME_SHA=$(ssh wooo@${TARGET_HOST} "docker exec '${PROMETHEUS_CONTAINER}' sha256sum '${PROMETHEUS_RUNTIME_RULES}' | awk '{print \$1}'") +if [ "$LOCAL_ALERTS_SHA" != "$REMOTE_RUNTIME_SHA" ]; then + echo "ERROR: Prometheus runtime rules hash 不一致 local=${LOCAL_ALERTS_SHA} runtime=${REMOTE_RUNTIME_SHA}" + exit 1 +fi +log "✅ Prometheus runtime rules hash 與 canonical source 一致" + +bash scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh +log "✅ Cold-start producer / active rules / Agent99 consumer scope verifier 通過" + +DEPLOY_VERIFIED=1 +log "Rollback receipts: ${TARGET_ALERTS_BACKUP_PATH}, ${TARGET_ALERTS_CANONICAL_BACKUP_PATH}" log "🎉 部署完成！所有關鍵規則已生效" diff --git a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py index 7e06ecab2..5ed821e5d 100644 --- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py +++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py @@ -20,6 +20,7 @@ RECOVERY_SCORECARD = ( ROOT / "scripts" / "reboot-recovery" / "full-stack-recovery-scorecard.sh" ) VERIFY_DEPLOY = ROOT / "scripts" / "reboot-recovery" / "verify-cold-start-monitor-deploy.sh" +DEPLOY_ALERTS = ROOT / "scripts" / "ops" / "deploy-alerts.sh" REPAIR_STARTUP_STUB = ( ROOT / "scripts" / "reboot-recovery" / "repair-110-startup-script-stub.sh" ) @@ -105,6 +106,20 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None: assert 'scope="110_120_121_188"' not in ansible_verifier +def test_alert_rule_deploy_is_preflighted_verified_and_rollback_capable() -> None: + deploy = DEPLOY_ALERTS.read_text(encoding="utf-8") + + assert "remote_promtool_preflight" in deploy + assert "promtool check rules" in deploy + assert "trap on_exit EXIT" in deploy + assert "rollback_deploy" in deploy + assert 'APPLY_STARTED=1' in deploy + assert 'DEPLOY_VERIFIED=1' in deploy + assert "PROMETHEUS_RUNTIME_RULES" in deploy + assert "verify-cold-start-monitor-deploy.sh" in deploy + assert "curl -fsS -X POST" in deploy + + def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None: text = COLD_START_CHECK.read_text(encoding="utf-8") From 3d8fa0200e113e4a61f16702fb91e5435a6a0858 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 02:15:44 +0800 Subject: [PATCH 5/5] fix(security): remove credentialed reboot SOP command --- docs/runbooks/REBOOT-RECOVERY-SOP.md | 3 ++- .../tests/test_reboot_p0_operational_contract.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index e80956cc1..a5e3da73a 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -479,7 +479,8 @@ curl -s --max-time 5 http://192.168.0.121:32334/api/v1/health 2>/dev/null | pyth ```bash # 自動化腳本通常已處理，但若沒有自動啟動: ssh ollama@192.168.0.188 "sudo /usr/local/bin/awoooi-startup.sh" -ssh wooo@192.168.0.110 "echo '0936223270' | sudo -S /usr/local/bin/awoooi-startup-110.sh" +ssh wooo@192.168.0.110 "sudo -n /usr/local/bin/awoooi-startup-110.sh" +# 若 sudo -n 不可用，改走 Agent99 allowlisted executor 或本機 console；禁止把密碼寫入 SOP、命令列或 evidence。 ``` **P1 (T+5~15 分鐘): K3s 叢集** diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index f4612c7e5..644f4c4f1 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -15,6 +15,13 @@ def read(path: str) -> str: return (ROOT / path).read_text(encoding="utf-8") +def test_reboot_sop_never_embeds_passworded_sudo_startup() -> None: + sop = read("docs/runbooks/REBOOT-RECOVERY-SOP.md") + + assert "sudo -n /usr/local/bin/awoooi-startup-110.sh" in sop + assert "| sudo -S /usr/local/bin/awoooi-startup-110.sh" not in sop + + def load_public_maintenance_probe() -> ModuleType: path = ROOT / "scripts/reboot-recovery/public-maintenance-fallback-probe.py" spec = importlib.util.spec_from_file_location(