From dc90bcc44e74aceb7db56a81e71c5cb86ecd9df6 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 01:00:38 +0800 Subject: [PATCH 1/7] chore(cd): deploy 76da612 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index ae2109247..5549a3367 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "5a593fba5e110288185c5498509b43c37bc52859" + value: "76da612d372401c5f8fe734ac7d70a55e0c1a9b2" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "5a593fba5e110288185c5498509b43c37bc52859" + value: "76da612d372401c5f8fe734ac7d70a55e0c1a9b2" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index a5ade2cf5..e8366803c 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "5a593fba5e110288185c5498509b43c37bc52859" + value: "76da612d372401c5f8fe734ac7d70a55e0c1a9b2" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 45cd29941..0b41e30ff 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "5a593fba5e110288185c5498509b43c37bc52859" + value: "76da612d372401c5f8fe734ac7d70a55e0c1a9b2" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index e5f9e0c44..07bdc9477 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:63787237ff5ace798b0978c0319969b9bff0a3cf44007dbd1c6eecadb4e9465e +- digest: sha256:5b976f5d9be1fe119e4137244ef1940a62f6ab0e55cbafdd1ec4fd569a9c586d name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:c454f0af821455d1b33947a706f3464ad06689cf6d3a681f0dcc5d420f90bad6 +- digest: sha256:88b5789fa2d147d37a1981a098705589f2d47435657cb98cf5cc5b004e12cded name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 8862da39297e91bd9b6cffedea4c1dd91422ebb8 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 00:42:23 +0800 Subject: [PATCH 2/7] fix(agent): route disk pressure to bounded repair --- .../services/awooop_ansible_audit_service.py | 107 ++++++- .../awooop_ansible_check_mode_service.py | 290 +++++++++++++++++- .../services/awooop_ansible_post_verifier.py | 30 ++ apps/api/src/services/telegram_gateway.py | 18 +- .../tests/test_ansible_verified_closure.py | 57 ++++ ...test_host_disk_pressure_ansible_catalog.py | 265 ++++++++++++++++ .../host-disk-pressure-bounded-cleanup.yml | 124 ++++++++ .../docker-disk-pressure-retention-cleanup.py | 197 +++++++++++- ..._docker_disk_pressure_retention_cleanup.py | 88 +++++- 9 files changed, 1145 insertions(+), 31 deletions(-) create mode 100644 apps/api/tests/test_host_disk_pressure_ansible_catalog.py create mode 100644 infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index bbe4a6c3a..7155ba6e1 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -417,6 +417,11 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "gitea_service", "systemd_control_plane", ], + "activation_keywords": [ + "host110sustainedmoderatepressure", + "host-pressure-controller", + "sustained pressure", + ], "supports_check_mode": True, "auto_apply_enabled": False, "approval_required": False, @@ -424,6 +429,74 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "no_write_only": True, "catalog_revision": "2026-07-14-host110-inputs-v2", }, + { + "catalog_id": "ansible:110-disk-pressure", + "playbook_path": ( + "infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml" + ), + "check_mode_playbook_path": ( + "infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml" + ), + "inventory_hosts": ["host_110"], + "domains": ["disk_pressure", "docker_artifacts", "host_110"], + "keywords": [ + "hostoutofdiskspace", + "hostdiskusagehigh", + "hostdiskusagecritical", + "hostdiskwillfillin24hours", + "disk_full", + "node-exporter-110", + "192.168.0.110", + ], + "activation_keyword_groups": [ + [ + "hostoutofdiskspace", + "hostdiskusagehigh", + "hostdiskusagecritical", + "hostdiskwillfillin24hours", + ], + ["node-exporter-110", "192.168.0.110", "host_110", "host110"], + ], + "supports_check_mode": True, + "auto_apply_enabled": True, + "approval_required": False, + "risk_level": "medium", + "catalog_revision": "2026-07-14-disk-pressure-bounded-v1", + }, + { + "catalog_id": "ansible:188-disk-pressure", + "playbook_path": ( + "infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml" + ), + "check_mode_playbook_path": ( + "infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml" + ), + "inventory_hosts": ["host_188"], + "domains": ["disk_pressure", "docker_artifacts", "host_188"], + "keywords": [ + "hostoutofdiskspace", + "hostdiskusagehigh", + "hostdiskusagecritical", + "hostdiskwillfillin24hours", + "disk_full", + "node-exporter-188", + "192.168.0.188", + ], + "activation_keyword_groups": [ + [ + "hostoutofdiskspace", + "hostdiskusagehigh", + "hostdiskusagecritical", + "hostdiskwillfillin24hours", + ], + ["node-exporter-188", "192.168.0.188", "host_188", "host188"], + ], + "supports_check_mode": True, + "auto_apply_enabled": True, + "approval_required": False, + "risk_level": "medium", + "catalog_revision": "2026-07-14-disk-pressure-bounded-v1", + }, { "catalog_id": "ansible:110-devops", "playbook_path": "infra/ansible/playbooks/110-node-exporter-repair.yml", @@ -437,9 +510,13 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "node_exporter", "node-exporter", ], - "activation_keywords": [ - "node-exporter-110", - "node exporter 110", + "activation_keyword_groups": [ + [ + "nodeexporterdown", + "nodeexporterscrapedown", + "node_exporter_down", + ], + ["node-exporter-110", "192.168.0.110", "host_110", "host110"], ], "supports_check_mode": True, "auto_apply_enabled": True, @@ -914,7 +991,15 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None matched_activation_keywords = [ keyword for keyword in activation_keywords if keyword in haystack ] - semantic_match = not activation_keywords or bool(matched_activation_keywords) + activation_keyword_groups = item.get("activation_keyword_groups") or [] + matched_activation_keyword_groups = [ + [keyword for keyword in group if keyword in haystack] + for group in activation_keyword_groups + ] + semantic_match = ( + (not activation_keywords or bool(matched_activation_keywords)) + and all(matched_activation_keyword_groups) + ) public_item = { key: value for key, value in item.items() @@ -939,6 +1024,9 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None "matched_keywords": matched, "semantic_match": True, "matched_activation_keywords": matched_activation_keywords, + "matched_activation_keyword_groups": ( + matched_activation_keyword_groups + ), }) else: unmatched.append(item["catalog_id"]) @@ -952,6 +1040,17 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None } +def get_ansible_catalog_candidates( + incident: dict[str, Any] | None, + drift: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + """Return current semantically matched catalog rows for replay routing.""" + + hints = _catalog_hints(incident, drift) + candidates = hints.get("candidates") + return [dict(row) for row in candidates or [] if isinstance(row, dict)] + + def build_ansible_truth( automation_ops: list[dict[str, Any]], *, diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 4363333e5..903899a83 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -19,7 +19,7 @@ from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any -from uuid import UUID, uuid4 +from uuid import NAMESPACE_URL, UUID, uuid4, uuid5 import structlog from sqlalchemy import select, text @@ -31,7 +31,10 @@ from src.models.approval import ApprovalStatus from src.services.ai_automation_runtime_contract import ( AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION, ) -from src.services.awooop_ansible_audit_service import get_ansible_catalog_item +from src.services.awooop_ansible_audit_service import ( + get_ansible_catalog_candidates, + get_ansible_catalog_item, +) from src.services.awooop_ansible_learning_writeback import ( canonical_ansible_playbook_id, ensure_ansible_rag_writeback, @@ -52,6 +55,7 @@ _CONTROLLED_RETRY_MAX_ATTEMPTS = 1 _CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS = 15 _CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS = 30 _STDIN_BOUNDARY_REPLAY_WINDOW_HOURS = 7 * 24 +_CATALOG_REPAIR_REPLAY_WINDOW_HOURS = 72 _STDIN_BOUNDARY_TERMINAL_PROJECTION_BATCH_LIMIT = 5 _EXECUTION_CAPABILITY_MIN_TTL_SECONDS = 300 _EXECUTION_CAPABILITY_MAX_TTL_SECONDS = 1_800 @@ -1062,9 +1066,28 @@ def _claim_from_stale_check_mode_row( or input_payload.get("incident_id") or "" ) - catalog_id = str(input_payload.get("catalog_id") or "") + previous_catalog_id = str(input_payload.get("catalog_id") or "") + catalog_id = previous_catalog_id + incident_routing_evaluated = bool( + row.get("incident_alertname") + or row.get("incident_alert_category") + or row.get("incident_affected_services") + ) + if incident_routing_evaluated: + candidates = get_ansible_catalog_candidates( + { + "alertname": row.get("incident_alertname"), + "alert_category": row.get("incident_alert_category"), + "affected_services": row.get("incident_affected_services") or [], + } + ) + if not candidates: + return None + catalog_id = str(candidates[0].get("catalog_id") or "") catalog_item = get_ansible_catalog_item(catalog_id) or {} - inventory_hosts = input_payload.get("inventory_hosts") + inventory_hosts = catalog_item.get("inventory_hosts") or input_payload.get( + "inventory_hosts" + ) source_playbook_path = str( input_payload.get("source_candidate_playbook_path") or input_payload.get("catalog_playbook_path") @@ -1106,6 +1129,13 @@ def _claim_from_stale_check_mode_row( ) except ValueError: return None + claim_input.update( + { + "previous_catalog_id": previous_catalog_id or None, + "catalog_route_changed": catalog_id != previous_catalog_id, + "incident_semantic_route_rechecked": incident_routing_evaluated, + } + ) return AnsibleCheckModeClaim( op_id=str(row.get("op_id") or ""), source_candidate_op_id=source_candidate_op_id, @@ -5663,6 +5693,10 @@ async def _send_controlled_apply_telegram_receipt( try: from src.services.telegram_gateway import get_telegram_gateway + effective_provider_delivery = str( + claim.input_payload.get("telegram_provider_delivery") + or provider_delivery + ) response = await get_telegram_gateway().send_controlled_apply_result_receipt( automation_run_id=str( claim.input_payload.get("automation_run_id") @@ -5680,7 +5714,7 @@ async def _send_controlled_apply_telegram_receipt( verifier_written=bool(writeback.get("verification")), learning_written=bool(writeback.get("learning")), execution_kind=execution_kind, - provider_delivery=provider_delivery, + provider_delivery=effective_provider_delivery, project_id=project_id, ) return bool( @@ -7654,9 +7688,10 @@ async def claim_catalog_drift_failed_check_modes( """Replay a failed check once when its canonical check playbook changed.""" claims: list[AnsibleCheckModeClaim] = [] - max_age_hours = ( + max_age_hours = max( candidate_max_age_hours - or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS + or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS, + _CATALOG_REPAIR_REPLAY_WINDOW_HOURS, ) async with get_db_context(project_id) as db: result = await db.execute( @@ -7668,10 +7703,20 @@ async def claim_catalog_drift_failed_check_modes( check_mode.input ->> 'incident_id', check_mode.incident_id::text ) AS incident_id, - check_mode.input + check_mode.input, + incident.alertname AS incident_alertname, + incident.alert_category AS incident_alert_category, + incident.affected_services AS incident_affected_services FROM automation_operation_log check_mode + JOIN incidents incident + ON incident.incident_id = coalesce( + check_mode.input ->> 'incident_id', + check_mode.incident_id::text + ) + AND incident.project_id = :project_id WHERE check_mode.operation_type = 'ansible_check_mode_executed' AND check_mode.status = 'failed' + AND upper(incident.status::text) NOT IN ('RESOLVED', 'CLOSED') AND check_mode.created_at >= NOW() - ( :candidate_max_age_hours * INTERVAL '1 hour' ) @@ -7686,6 +7731,7 @@ async def claim_catalog_drift_failed_check_modes( FOR UPDATE SKIP LOCKED """), { + "project_id": project_id, "candidate_max_age_hours": max(1, max_age_hours), "scan_limit": min(100, max(10, limit * 10)), }, @@ -7733,6 +7779,8 @@ async def claim_catalog_drift_failed_check_modes( "replay_of_check_mode_op_id": str(row.get("op_id") or ""), "previous_check_mode_playbook_path": previous_check_path, "previous_catalog_revision": previous_catalog_revision or None, + "historical_projection_backfill": True, + "telegram_provider_delivery": "shadow_only", } inserted = await db.execute( text(""" @@ -7761,6 +7809,9 @@ async def claim_catalog_drift_failed_check_modes( = :catalog_replay_revision AND existing_replay.input ->> 'catalog_drift_replay' = 'true' + AND existing_replay.input + ->> 'replay_of_check_mode_op_id' + = :failed_check_mode_op_id ) RETURNING op_id """), @@ -7786,6 +7837,7 @@ async def claim_catalog_drift_failed_check_modes( "parent_op_id": canonical_claim.source_candidate_op_id, "catalog_id": canonical_claim.catalog_id, "catalog_replay_revision": catalog_replay_revision, + "failed_check_mode_op_id": str(row.get("op_id") or ""), "tags": [ "ansible", "check_mode", @@ -7939,6 +7991,197 @@ async def claim_catalog_drift_failed_applies( return claims +async def claim_semantically_misrouted_verified_applies( + *, + project_id: str = "awoooi", + limit: int = 1, + window_hours: int = _CATALOG_REPAIR_REPLAY_WINDOW_HOURS, +) -> list[AnsibleCheckModeClaim]: + """Re-verify terminal applies whose catalog no longer matches the alert. + + Resolved incidents remain eligible because this path corrects false terminal + attribution; it does not reopen incidents from an unresolved alert signal. + """ + + claims: list[AnsibleCheckModeClaim] = [] + async with get_db_context(project_id) as db: + result = await db.execute( + text(""" + SELECT + source_check.op_id, + source_check.parent_op_id, + incident.incident_id, + source_check.input, + incident.alertname AS incident_alertname, + incident.alert_category AS incident_alert_category, + incident.affected_services AS incident_affected_services, + apply.op_id::text AS previous_apply_op_id + FROM automation_operation_log apply + JOIN automation_operation_log source_check + ON source_check.op_id = apply.parent_op_id + AND source_check.operation_type + = 'ansible_check_mode_executed' + JOIN incidents incident + ON incident.incident_id = coalesce( + apply.input ->> 'incident_id', + apply.incident_id::text + ) + AND incident.project_id = :project_id + WHERE apply.operation_type = 'ansible_apply_executed' + AND apply.status = 'success' + AND apply.output ->> 'independent_post_verifier_passed' + = 'true' + AND apply.created_at >= NOW() - ( + :window_hours * INTERVAL '1 hour' + ) + ORDER BY apply.created_at DESC + LIMIT :scan_limit + FOR UPDATE OF apply SKIP LOCKED + """), + { + "project_id": project_id, + "window_hours": max(1, min(window_hours, 7 * 24)), + "scan_limit": min(100, max(10, limit * 10)), + }, + ) + seen_incident_ids: set[str] = set() + for row_value in result.mappings().all(): + row = dict(row_value) + incident_id = str(row.get("incident_id") or "") + if not incident_id or incident_id in seen_incident_ids: + continue + seen_incident_ids.add(incident_id) + previous_input = _json_loads(row.get("input")) + previous_catalog_id = str( + previous_input.get("catalog_id") or "" + ) + canonical_claim = _claim_from_stale_check_mode_row(row) + if ( + canonical_claim is None + or not previous_catalog_id + or canonical_claim.catalog_id == previous_catalog_id + ): + continue + current_revision = str( + canonical_claim.input_payload.get("catalog_revision") or "" + ) + previous_apply_op_id = str( + row.get("previous_apply_op_id") or "" + ) + if not current_revision or not previous_apply_op_id: + continue + semantic_repair_run_id = str( + uuid5( + NAMESPACE_URL, + ( + "awoooi:ansible-semantic-repair:" + f"{project_id}:{previous_apply_op_id}:" + f"{current_revision}" + ), + ) + ) + replay_input = { + **canonical_claim.input_payload, + "automation_run_id": semantic_repair_run_id, + "semantic_route_reconciliation": True, + "semantic_repair_of_apply_op_id": previous_apply_op_id, + "catalog_replay_revision": current_revision, + "historical_projection_backfill": True, + "telegram_provider_delivery": "shadow_only", + "apply_idempotency_key": ( + "ansible-semantic-repair:" + f"{previous_apply_op_id}:{current_revision}" + ), + "bounded_execution_scope": ( + "semantic_catalog_reconciliation_single_apply" + ), + } + inserted = await db.execute( + text(""" + INSERT INTO automation_operation_log ( + operation_type, actor, status, incident_id, + input, output, dry_run_result, + parent_op_id, tags + ) SELECT + 'ansible_check_mode_executed', + 'ansible_check_mode_worker', + 'pending', + :incident_db_id, + CAST(:input AS jsonb), + '{}'::jsonb, + CAST(:dry_run_result AS jsonb), + CAST(:parent_op_id AS uuid), + :tags + WHERE NOT EXISTS ( + SELECT 1 + FROM automation_operation_log existing_replay + WHERE existing_replay.operation_type + = 'ansible_check_mode_executed' + AND existing_replay.input + ->> 'semantic_repair_of_apply_op_id' + = :previous_apply_op_id + AND existing_replay.input + ->> 'catalog_replay_revision' + = :catalog_replay_revision + ) + RETURNING op_id + """), + { + "incident_db_id": _automation_operation_log_incident_id( + canonical_claim.incident_id + ), + "input": json.dumps(replay_input, ensure_ascii=False), + "dry_run_result": json.dumps( + { + "check_mode_executed": False, + "apply_executed": False, + "claim_state": ( + "semantic_catalog_reconciliation" + ), + "controlled_apply_allowed": bool( + replay_input.get("controlled_apply_allowed") + ), + }, + ensure_ascii=False, + ), + "parent_op_id": ( + canonical_claim.source_candidate_op_id + ), + "previous_apply_op_id": previous_apply_op_id, + "catalog_replay_revision": current_revision, + "tags": [ + "ansible", + "check_mode", + "pending", + "semantic_catalog_reconciliation", + ], + }, + ) + op_id = inserted.scalar_one_or_none() + if op_id is None: + continue + claims.append( + AnsibleCheckModeClaim( + op_id=str(op_id), + source_candidate_op_id=( + canonical_claim.source_candidate_op_id + ), + incident_id=canonical_claim.incident_id, + catalog_id=canonical_claim.catalog_id, + playbook_path=canonical_claim.playbook_path, + apply_playbook_path=( + canonical_claim.apply_playbook_path + ), + inventory_hosts=canonical_claim.inventory_hosts, + risk_level=canonical_claim.risk_level, + input_payload=replay_input, + ) + ) + if len(claims) >= max(1, limit): + break + return claims + + async def recent_ansible_transport_blockers( *, project_id: str = "awoooi", @@ -8716,6 +8959,30 @@ async def run_pending_check_modes_once( 0, remaining_limit - len(stdin_boundary_replay_claims), ) + semantic_catalog_reconciliation_claims: list[ + AnsibleCheckModeClaim + ] = [] + semantic_catalog_reconciliation_error: str | None = None + if remaining_limit: + try: + semantic_catalog_reconciliation_claims = ( + await claim_semantically_misrouted_verified_applies( + project_id=project_id, + limit=remaining_limit, + ) + ) + except Exception as exc: + semantic_catalog_reconciliation_error = type(exc).__name__ + logger.warning( + "ansible_semantic_catalog_reconciliation_claim_failed", + project_id=project_id, + error_type=semantic_catalog_reconciliation_error, + fresh_candidate_claim_continues=True, + ) + remaining_limit = max( + 0, + remaining_limit - len(semantic_catalog_reconciliation_claims), + ) failed_apply_catalog_replay_claims: list[AnsibleCheckModeClaim] = [] failed_apply_catalog_replay_error: str | None = None if remaining_limit: @@ -8775,6 +9042,7 @@ async def run_pending_check_modes_once( claims = [ *reclaimed_claims, *stdin_boundary_replay_claims, + *semantic_catalog_reconciliation_claims, *failed_apply_catalog_replay_claims, *catalog_replay_claims, *fresh_claims, @@ -8924,6 +9192,12 @@ async def run_pending_check_modes_once( stdin_boundary_replay_terminal_written ), "stdin_boundary_replay_error": stdin_boundary_replay_error, + "semantic_catalog_reconciled": len( + semantic_catalog_reconciliation_claims + ), + "semantic_catalog_reconciliation_error": ( + semantic_catalog_reconciliation_error + ), "failed_apply_catalog_replayed": len( failed_apply_catalog_replay_claims ), diff --git a/apps/api/src/services/awooop_ansible_post_verifier.py b/apps/api/src/services/awooop_ansible_post_verifier.py index be5806d5a..c822f6e16 100644 --- a/apps/api/src/services/awooop_ansible_post_verifier.py +++ b/apps/api/src/services/awooop_ansible_post_verifier.py @@ -98,6 +98,36 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = { "test -s /home/wooo/node_exporter_textfiles/docker_stats.prom", ), ), + "ansible:110-disk-pressure": ( + AssetPostcondition( + "host_110_root_disk_below_alert_threshold", + "metric", + "host_110", + "usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); " + "test -n \"$usage\" && test \"$usage\" -lt 85", + ), + AssetPostcondition( + "host_110_docker_runtime_after_cleanup", + "service", + "host_110", + "docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0", + ), + ), + "ansible:188-disk-pressure": ( + AssetPostcondition( + "host_188_root_disk_below_alert_threshold", + "metric", + "host_188", + "usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); " + "test -n \"$usage\" && test \"$usage\" -lt 85", + ), + AssetPostcondition( + "host_188_docker_runtime_after_cleanup", + "service", + "host_188", + "docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0", + ), + ), "ansible:110-devops": ( AssetPostcondition( "host_110_node_exporter_container", diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index a13e22dfb..9b3dd07f8 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -5312,9 +5312,9 @@ class TelegramGateway: row = existing.mappings().one_or_none() if row is not None: existing_row = dict(row) - elif failure_fingerprint: + elif force_suppress or failure_fingerprint: should_suppress = force_suppress - if not should_suppress: + if not should_suppress and failure_fingerprint: group_reservation = await db.execute( text(""" SELECT message_id::text AS message_id @@ -10370,6 +10370,20 @@ class TelegramGateway: "km_rag_playbook", ], } + elif provider_delivery == "shadow_only": + source_extra["notification_policy"] = { + "policy_version": "telegram_historical_repair_shadow_v1", + "failure_fingerprint": "", + "group_scope": "historical_projection_backfill", + "provider_delivery": "shadow_only", + "disposition": "suppressed", + "provider_send_performed": False, + "details_retained_in": [ + "automation_operation_log", + "awooop_runs", + "km_rag_playbook", + ], + } if no_write_replay: lines = [ f"{html.escape(title)}", diff --git a/apps/api/tests/test_ansible_verified_closure.py b/apps/api/tests/test_ansible_verified_closure.py index fd7bcd5e9..de64fff4a 100644 --- a/apps/api/tests/test_ansible_verified_closure.py +++ b/apps/api/tests/test_ansible_verified_closure.py @@ -992,6 +992,63 @@ async def test_historical_no_write_backfill_is_provider_silent( assert policy["suppression_reason"] == "historical_projection_backfill" +@pytest.mark.asyncio +async def test_historical_verified_repair_is_provider_silent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + db = _SequenceDB( + _MappingResult(), + _MappingResult(row=None), + ) + + @asynccontextmanager + async def fake_get_db_context(_project_id): + yield db + + record_outbound = AsyncMock( + return_value="00000000-0000-0000-0000-000000000498" + ) + monkeypatch.setattr("src.db.base.get_db_context", fake_get_db_context) + monkeypatch.setattr( + "src.services.channel_hub.record_outbound_message", + record_outbound, + ) + gateway = TelegramGateway() + identity = { + "project_id": "awoooi", + "automation_run_id": "00000000-0000-0000-0000-000000000402", + "incident_id": "INC-20260711-HISTORICAL-SUCCESS", + "apply_op_id": "00000000-0000-0000-0000-000000000404", + } + source_extra = { + "callback_reply": { + "action": "controlled_apply_result", + "execution_kind": "controlled_apply", + **identity, + }, + "notification_policy": { + "failure_fingerprint": "", + "disposition": "suppressed", + "provider_delivery": "shadow_only", + }, + } + + reservation = await gateway._reserve_controlled_apply_result_outbound( + identity=identity, + payload={"chat_id": "sre-chat", "text": "historical repair closed"}, + source_envelope_extra=source_extra, + ) + + assert reservation["status"] == "suppressed" + assert len(db.statements) == 2 + record_kwargs = record_outbound.await_args.kwargs + assert record_kwargs["send_status"] == "shadow" + assert record_kwargs["provider_message_id"] is None + policy = record_kwargs["source_envelope"]["notification_policy"] + assert policy["suppression_reason"] == "historical_projection_backfill" + assert policy["provider_send_performed"] is False + + def test_controlled_result_uses_durable_reservation_and_single_reconcile_entry() -> None: reserve_source = inspect.getsource( TelegramGateway._reserve_controlled_apply_result_outbound diff --git a/apps/api/tests/test_host_disk_pressure_ansible_catalog.py b/apps/api/tests/test_host_disk_pressure_ansible_catalog.py new file mode 100644 index 000000000..d373e032c --- /dev/null +++ b/apps/api/tests/test_host_disk_pressure_ansible_catalog.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +import inspect +from contextlib import asynccontextmanager +from pathlib import Path +from uuid import UUID + +import pytest +import yaml + +from src.services.awooop_ansible_audit_service import ( + _catalog_hints, + get_ansible_catalog_item, +) +from src.services.awooop_ansible_check_mode_service import ( + _claim_from_stale_check_mode_row, + _send_controlled_apply_telegram_receipt, + claim_catalog_drift_failed_check_modes, + claim_semantically_misrouted_verified_applies, + run_pending_check_modes_once, +) +from src.services.awooop_ansible_post_verifier import postconditions_for_catalog + +ROOT = Path(__file__).resolve().parents[3] +PLAYBOOK = ( + ROOT + / "infra" + / "ansible" + / "playbooks" + / "host-disk-pressure-bounded-cleanup.yml" +) + + +def _disk_incident(host: str) -> dict: + return { + "alertname": "HostOutOfDiskSpace", + "alert_category": "host_resource", + "affected_services": [f"node-exporter-{host}"], + } + + +def test_disk_alert_routes_to_the_matching_host_catalog() -> None: + host110 = _catalog_hints(_disk_incident("110"), None) + host188 = _catalog_hints(_disk_incident("188"), None) + + assert [row["catalog_id"] for row in host110["candidates"]] == [ + "ansible:110-disk-pressure" + ] + assert [row["catalog_id"] for row in host188["candidates"]] == [ + "ansible:188-disk-pressure" + ] + assert all( + row["catalog_id"] != "ansible:110-devops" + for row in [*host110["candidates"], *host188["candidates"]] + ) + + +def test_node_exporter_repair_requires_a_down_signal_not_a_disk_alert() -> None: + disk_hints = _catalog_hints(_disk_incident("110"), None) + down_hints = _catalog_hints( + { + "alertname": "NodeExporterDown", + "affected_services": ["node-exporter-110"], + }, + None, + ) + + assert "ansible:110-devops" not in { + row["catalog_id"] for row in disk_hints["candidates"] + } + assert [row["catalog_id"] for row in down_hints["candidates"]] == [ + "ansible:110-devops" + ] + + +def test_disk_cleanup_playbook_excludes_destructive_asset_classes() -> None: + payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8")) + source = PLAYBOOK.read_text(encoding="utf-8").lower() + + assert payload[0]["hosts"] == "devops:ai_web" + assert payload[0]["become"] is False + assert "docker-disk-pressure-retention-cleanup.py" in source + assert "--apply" in source + assert "--max-image-removals" in source + assert "single_apply_image_removal_is_count_bounded" in source + assert "--include-builder-cache" not in source + for forbidden in ( + "docker system prune", + "docker volume prune", + "truncate -s", + "rm -rf", + "reboot", + "systemctl restart", + ): + assert forbidden not in source + + +def test_stale_generic_host188_row_is_semantically_remapped() -> None: + claim = _claim_from_stale_check_mode_row( + { + "op_id": "00000000-0000-0000-0000-000000000188", + "parent_op_id": "00000000-0000-0000-0000-000000000100", + "incident_id": "INC-DISK-188", + "incident_alertname": "HostOutOfDiskSpace", + "incident_alert_category": "host_resource", + "incident_affected_services": ["node-exporter-188"], + "input": { + "source_candidate_op_id": ( + "00000000-0000-0000-0000-000000000100" + ), + "incident_id": "INC-DISK-188", + "catalog_id": "ansible:110-devops", + "catalog_playbook_path": "infra/ansible/playbooks/110-devops.yml", + "playbook_path": "infra/ansible/playbooks/110-devops.yml", + "inventory_hosts": ["host_110"], + "risk_level": "medium", + }, + } + ) + + assert claim is not None + assert claim.catalog_id == "ansible:188-disk-pressure" + assert claim.inventory_hosts == ("host_188",) + assert claim.input_payload["catalog_route_changed"] is True + assert claim.input_payload["incident_semantic_route_rechecked"] is True + + +def test_disk_catalogs_have_independent_threshold_verifiers() -> None: + for host in ("110", "188"): + catalog_id = f"ansible:{host}-disk-pressure" + catalog = get_ansible_catalog_item(catalog_id) + conditions = postconditions_for_catalog(catalog_id) + + assert catalog is not None + assert catalog["auto_apply_enabled"] is True + assert catalog["risk_level"] == "medium" + assert len(conditions) == 2 + assert any("df -P /" in condition.probe for condition in conditions) + assert any("docker info" in condition.probe for condition in conditions) + + +def test_historical_catalog_replay_is_per_failed_row_and_provider_silent() -> None: + claim_source = inspect.getsource(claim_catalog_drift_failed_check_modes) + send_source = inspect.getsource(_send_controlled_apply_telegram_receipt) + + assert "replay_of_check_mode_op_id" in claim_source + assert "failed_check_mode_op_id" in claim_source + assert '"historical_projection_backfill": True' in claim_source + assert '"telegram_provider_delivery": "shadow_only"' in claim_source + assert "telegram_provider_delivery" in send_source + + +def test_verified_wrong_catalog_is_reconciled_before_failure_replays() -> None: + claim_source = inspect.getsource( + claim_semantically_misrouted_verified_applies + ) + worker_source = inspect.getsource(run_pending_check_modes_once) + + assert "semantic_route_reconciliation" in claim_source + assert "semantic_repair_of_apply_op_id" in claim_source + assert '"telegram_provider_delivery": "shadow_only"' in claim_source + assert "independent_post_verifier_passed" in claim_source + assert "seen_incident_ids" in claim_source + assert "RESOLVED" not in claim_source + assert worker_source.index( + "claim_semantically_misrouted_verified_applies" + ) < worker_source.index("claim_catalog_drift_failed_applies") + assert "semantic_catalog_reconciled" in worker_source + + +@pytest.mark.asyncio +async def test_verified_wrong_catalog_enqueues_one_shadow_disk_repair( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import awooop_ansible_check_mode_service as service + + previous_apply_op_id = "00000000-0000-0000-0000-000000000202" + + class Result: + def __init__(self, *, rows=None, scalar=None): + self._rows = rows or [] + self._scalar = scalar + + def mappings(self): + return self + + def all(self): + return self._rows + + def scalar_one_or_none(self): + return self._scalar + + class DB: + def __init__(self): + self.statements: list[str] = [] + + async def execute(self, statement, _params): + self.statements.append(str(statement)) + if len(self.statements) == 1: + return Result( + rows=[ + { + "op_id": ( + "00000000-0000-0000-0000-000000000201" + ), + "parent_op_id": ( + "00000000-0000-0000-0000-000000000200" + ), + "incident_id": "INC-DISK-188-VERIFIED-WRONG", + "incident_alertname": "HostOutOfDiskSpace", + "incident_alert_category": "host_resource", + "incident_affected_services": [ + "node-exporter-188" + ], + "previous_apply_op_id": previous_apply_op_id, + "input": { + "source_candidate_op_id": ( + "00000000-0000-0000-0000-000000000200" + ), + "incident_id": ( + "INC-DISK-188-VERIFIED-WRONG" + ), + "catalog_id": "ansible:110-devops", + "catalog_playbook_path": ( + "infra/ansible/playbooks/" + "110-node-exporter-repair.yml" + ), + "playbook_path": ( + "infra/ansible/playbooks/" + "110-node-exporter-repair.yml" + ), + "inventory_hosts": ["host_110"], + "risk_level": "low", + }, + } + ] + ) + return Result( + scalar="00000000-0000-0000-0000-000000000203" + ) + + db = DB() + + @asynccontextmanager + async def fake_get_db_context(_project_id): + yield db + + monkeypatch.setattr(service, "get_db_context", fake_get_db_context) + + claims = await claim_semantically_misrouted_verified_applies(limit=1) + + assert len(claims) == 1 + claim = claims[0] + assert claim.catalog_id == "ansible:188-disk-pressure" + assert claim.inventory_hosts == ("host_188",) + assert claim.input_payload["semantic_repair_of_apply_op_id"] == ( + previous_apply_op_id + ) + assert UUID(claim.input_payload["automation_run_id"]) + assert claim.input_payload["automation_run_id"] != ( + "00000000-0000-0000-0000-000000000200" + ) + assert claim.input_payload["telegram_provider_delivery"] == "shadow_only" + assert "DISTINCT ON" not in db.statements[0] + assert "FOR UPDATE OF apply SKIP LOCKED" in db.statements[0] diff --git a/infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml b/infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml new file mode 100644 index 000000000..18bee4b32 --- /dev/null +++ b/infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml @@ -0,0 +1,124 @@ +--- +- name: Host disk pressure bounded Docker artifact cleanup + hosts: devops:ai_web + gather_facts: false + become: false + serial: 1 + any_errors_fatal: true + + vars: + disk_cleanup_script: >- + {{ playbook_dir }}/../../../scripts/ops/docker-disk-pressure-retention-cleanup.py + disk_cleanup_gate_percent: 85 + disk_cleanup_min_age_hours: 24 + disk_cleanup_keep_dangling_newest: >- + {{ 3 if inventory_hostname == 'host_188' else 20 }} + disk_cleanup_include_tagged_images: >- + {{ inventory_hostname == 'host_188' }} + disk_cleanup_tagged_min_age_hours: 168 + disk_cleanup_keep_tagged_newest_per_repository: 3 + disk_cleanup_max_image_removals: >- + {{ 50 if inventory_hostname == 'host_188' else 100 }} + + tasks: + - name: Read root filesystem usage before cleanup + ansible.builtin.command: + argv: + - df + - -P + - / + register: disk_before + check_mode: false + changed_when: false + + - name: Normalize the disk pressure gate + ansible.builtin.set_fact: + disk_before_used_percent: >- + {{ disk_before.stdout_lines[-1].split()[4] | replace('%', '') | int }} + disk_cleanup_should_apply: >- + {{ + (not ansible_check_mode) + and ( + disk_before.stdout_lines[-1].split()[4] + | replace('%', '') + | int + ) >= disk_cleanup_gate_percent + }} + changed_when: false + + - name: Plan or apply bounded unreferenced image cleanup + ansible.builtin.script: + cmd: >- + {{ disk_cleanup_script }} + --host-label {{ inventory_hostname }} + --min-age-hours {{ disk_cleanup_min_age_hours }} + --keep-dangling-newest {{ disk_cleanup_keep_dangling_newest }} + --tagged-min-age-hours {{ disk_cleanup_tagged_min_age_hours }} + --keep-tagged-newest-per-repository + {{ disk_cleanup_keep_tagged_newest_per_repository }} + --max-image-removals {{ disk_cleanup_max_image_removals }} + {{ '--include-tagged-images' if disk_cleanup_include_tagged_images else '' }} + {{ '--apply' if disk_cleanup_should_apply else '' }} + register: disk_cleanup_command + check_mode: false + changed_when: >- + disk_cleanup_should_apply + and ((disk_cleanup_command.stdout | from_json).removed_image_ids | length > 0) + + - name: Parse the bounded cleanup receipt + ansible.builtin.set_fact: + disk_cleanup_receipt: "{{ disk_cleanup_command.stdout | from_json }}" + changed_when: false + + - name: Enforce the cleanup safety boundary + ansible.builtin.assert: + that: + - not disk_cleanup_receipt.boundaries.touches_containers + - not disk_cleanup_receipt.boundaries.touches_volumes + - not disk_cleanup_receipt.boundaries.touches_databases + - not disk_cleanup_receipt.boundaries.touches_backups + - not disk_cleanup_receipt.boundaries.uses_docker_system_prune + - disk_cleanup_receipt.boundaries.removes_only_container_unreferenced_images + - disk_cleanup_receipt.boundaries.tagged_image_cleanup_preserves_repository_rollback_window + - disk_cleanup_receipt.boundaries.single_apply_image_removal_is_count_bounded + - >- + (disk_cleanup_receipt.removed_image_ids | length) + <= (disk_cleanup_max_image_removals | int) + fail_msg: bounded_disk_cleanup_boundary_failed + success_msg: bounded_disk_cleanup_boundary_verified + changed_when: false + + - name: Read root filesystem usage after cleanup + ansible.builtin.command: + argv: + - df + - -P + - / + register: disk_after + check_mode: false + changed_when: false + + - name: Emit public-safe disk cleanup receipt + ansible.builtin.debug: + msg: + schema_version: host_disk_pressure_bounded_cleanup_v1 + host: "{{ inventory_hostname }}" + requested_mode: "{{ 'apply' if not ansible_check_mode else 'check' }}" + runtime_apply_executed: "{{ disk_cleanup_should_apply }}" + disk_before_used_percent: "{{ disk_before_used_percent }}" + disk_after_used_percent: >- + {{ disk_after.stdout_lines[-1].split()[4] | replace('%', '') | int }} + dangling_candidate_count: >- + {{ disk_cleanup_receipt.dangling_image_removal_plan.count }} + tagged_candidate_count: >- + {{ disk_cleanup_receipt.tagged_image_removal_plan.count }} + eligible_candidate_count: >- + {{ disk_cleanup_receipt.eligible_image_removal_plan.count }} + max_image_removals: "{{ disk_cleanup_max_image_removals }}" + removed_image_count: "{{ disk_cleanup_receipt.removed_image_ids | length }}" + touches_containers: false + touches_volumes: false + touches_databases: false + touches_backups: false + uses_docker_system_prune: false + secret_value_exposed: false diff --git a/scripts/ops/docker-disk-pressure-retention-cleanup.py b/scripts/ops/docker-disk-pressure-retention-cleanup.py index b8722df98..106422c87 100755 --- a/scripts/ops/docker-disk-pressure-retention-cleanup.py +++ b/scripts/ops/docker-disk-pressure-retention-cleanup.py @@ -2,9 +2,10 @@ """Bounded Docker disk-pressure cleanup for host reboot recovery. This controller intentionally avoids `docker system prune`, volumes, containers, -running images, databases, backups, and logs. It only removes dangling images -that are not referenced by any container, and can optionally run a bounded -BuildKit cache cleanup with an explicit keep-storage floor. +running images, databases, backups, and logs. It removes only images that are +not referenced by any container. Tagged-image cleanup is explicit, age-bounded, +and preserves a per-repository rollback window. BuildKit cleanup is separately +opt-in and preserves an explicit storage floor. """ from __future__ import annotations @@ -13,14 +14,18 @@ import argparse import json import subprocess import sys +from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import Any, Iterable - +from typing import Any +UTC = timezone.utc # noqa: UP017 - controller still runs on Python 3.9 hosts. DEFAULT_MIN_AGE_HOURS = 24 DEFAULT_KEEP_DANGLING_NEWEST = 20 +DEFAULT_TAGGED_MIN_AGE_HOURS = 168 +DEFAULT_KEEP_TAGGED_NEWEST_PER_REPOSITORY = 3 +DEFAULT_MAX_IMAGE_REMOVALS = 100 DEFAULT_BUILDER_KEEP_STORAGE = "30GB" @@ -42,6 +47,30 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--host-label", default="") parser.add_argument("--min-age-hours", type=int, default=DEFAULT_MIN_AGE_HOURS) parser.add_argument("--keep-dangling-newest", type=int, default=DEFAULT_KEEP_DANGLING_NEWEST) + parser.add_argument( + "--include-tagged-images", + action="store_true", + help=( + "Also remove old, container-unreferenced tagged images while preserving " + "the newest images for every repository." + ), + ) + parser.add_argument( + "--tagged-min-age-hours", + type=int, + default=DEFAULT_TAGGED_MIN_AGE_HOURS, + ) + parser.add_argument( + "--keep-tagged-newest-per-repository", + type=int, + default=DEFAULT_KEEP_TAGGED_NEWEST_PER_REPOSITORY, + ) + parser.add_argument( + "--max-image-removals", + type=int, + default=DEFAULT_MAX_IMAGE_REMOVALS, + help="Maximum image IDs removed by one controlled apply.", + ) parser.add_argument( "--skip-dangling-images", action="store_true", @@ -102,8 +131,8 @@ def parse_docker_datetime(value: str) -> datetime: text = f"{head}.{frac_text}{suffix}" parsed = datetime.fromisoformat(text) if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc) + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) def chunked(values: list[str], size: int) -> Iterable[list[str]]: @@ -170,6 +199,45 @@ def get_dangling_images(docker_bin: str) -> list[ImageInfo]: return images +def get_all_images(docker_bin: str) -> list[ImageInfo]: + image_ids = { + normalize_image_id(value) + for value in docker( + ["image", "ls", "--all", "--quiet", "--no-trunc"], + docker_bin, + ).stdout.split() + if normalize_image_id(value) + } + images: list[ImageInfo] = [] + for group in chunked(sorted(image_ids), 100): + result = docker(["image", "inspect", *group], docker_bin) + payload = json.loads(result.stdout or "[]") + for item in payload: + image_id = normalize_image_id(str(item.get("Id") or "")) + if not image_id: + continue + tags = item.get("RepoTags") or [] + images.append( + ImageInfo( + image_id=image_id, + created_at=parse_docker_datetime(str(item.get("Created") or "")), + size_bytes=int(item.get("Size") or 0), + repo_tags=tuple(str(tag) for tag in tags if tag), + ) + ) + return images + + +def repository_from_tag(tag: str) -> str: + value = str(tag or "").strip() + if "@" in value: + return value.split("@", 1)[0] + tail = value.rsplit("/", 1)[-1] + if ":" not in tail: + return value + return value.rsplit(":", 1)[0] + + def select_dangling_image_removals( images: list[ImageInfo], protected_ids: set[str], @@ -192,6 +260,45 @@ def select_dangling_image_removals( return sorted(dangling, key=lambda image: image.created_at) +def select_tagged_image_removals( + images: list[ImageInfo], + protected_ids: set[str], + *, + now: datetime, + min_age_hours: int, + keep_newest_per_repository: int, +) -> list[ImageInfo]: + unreferenced = [ + image + for image in images + if normalize_image_id(image.image_id) not in protected_ids + and image.repo_tags + ] + repository_images: dict[str, list[ImageInfo]] = {} + for image in unreferenced: + for repository in { + repository_from_tag(tag) for tag in image.repo_tags if tag + }: + repository_images.setdefault(repository, []).append(image) + + retained_ids: set[str] = set() + for repository_rows in repository_images.values(): + repository_rows.sort(key=lambda image: image.created_at, reverse=True) + retained_ids.update( + normalize_image_id(image.image_id) + for image in repository_rows[:keep_newest_per_repository] + ) + + cutoff_seconds = min_age_hours * 3600 + selected = [ + image + for image in unreferenced + if normalize_image_id(image.image_id) not in retained_ids + and (now - image.created_at).total_seconds() >= cutoff_seconds + ] + return sorted(selected, key=lambda image: image.created_at) + + def summarize_images(images: list[ImageInfo]) -> dict[str, Any]: return { "count": len(images), @@ -202,13 +309,29 @@ def summarize_images(images: list[ImageInfo]) -> dict[str, Any]: } +def limit_image_removals( + images: list[ImageInfo], + *, + max_removals: int, +) -> list[ImageInfo]: + return sorted(images, key=lambda image: image.created_at)[:max_removals] + + def remove_images(images: list[ImageInfo], docker_bin: str) -> list[str]: removed: list[str] = [] - for group in chunked([image.image_id for image in images], 25): - if not group: - continue + dangling_ids = [image.image_id for image in images if not image.repo_tags] + for group in chunked(dangling_ids, 25): docker(["image", "rm", *group], docker_bin) removed.extend(group) + tagged_images = [image for image in images if image.repo_tags] + tagged_references = [ + reference + for image in tagged_images + for reference in image.repo_tags + ] + for group in chunked(tagged_references, 25): + docker(["image", "rm", *group], docker_bin) + removed.extend(image.image_id for image in tagged_images) return removed @@ -227,11 +350,11 @@ def builder_prune_command(args: argparse.Namespace) -> list[str]: def build_receipt(args: argparse.Namespace) -> dict[str, Any]: - now = datetime.now(timezone.utc) + now = datetime.now(UTC) before = current_disk_bytes(args.disk_path) protected_ids = get_container_image_ids(args.docker_bin) dangling_images = get_dangling_images(args.docker_bin) - removal_candidates = ( + dangling_removal_candidates = ( [] if args.skip_dangling_images else select_dangling_image_removals( @@ -242,6 +365,29 @@ def build_receipt(args: argparse.Namespace) -> dict[str, Any]: keep_newest=args.keep_dangling_newest, ) ) + all_images = get_all_images(args.docker_bin) if args.include_tagged_images else [] + tagged_removal_candidates = ( + select_tagged_image_removals( + all_images, + protected_ids, + now=now, + min_age_hours=args.tagged_min_age_hours, + keep_newest_per_repository=args.keep_tagged_newest_per_repository, + ) + if args.include_tagged_images + else [] + ) + candidates_by_id = { + normalize_image_id(image.image_id): image + for image in [*dangling_removal_candidates, *tagged_removal_candidates] + } + eligible_removal_candidates = sorted( + candidates_by_id.values(), key=lambda image: image.created_at + ) + removal_candidates = limit_image_removals( + eligible_removal_candidates, + max_removals=args.max_image_removals, + ) receipt: dict[str, Any] = { "schema_version": "awoooi_docker_disk_pressure_retention_cleanup_v1", "generated_at": now.isoformat(), @@ -254,12 +400,21 @@ def build_receipt(args: argparse.Namespace) -> dict[str, Any]: "touches_databases": False, "touches_backups": False, "uses_docker_system_prune": False, - "removes_only_unreferenced_dangling_images": True, + "removes_only_container_unreferenced_images": True, + "tagged_image_cleanup_requires_explicit_flag": True, + "tagged_image_cleanup_preserves_repository_rollback_window": True, + "single_apply_image_removal_is_count_bounded": True, "builder_cache_cleanup_requires_explicit_flag": True, }, "parameters": { "min_age_hours": args.min_age_hours, "keep_dangling_newest": args.keep_dangling_newest, + "include_tagged_images": args.include_tagged_images, + "tagged_min_age_hours": args.tagged_min_age_hours, + "keep_tagged_newest_per_repository": ( + args.keep_tagged_newest_per_repository + ), + "max_image_removals": args.max_image_removals, "include_builder_cache": args.include_builder_cache, "builder_keep_storage": args.builder_keep_storage, "skip_dangling_images": args.skip_dangling_images, @@ -267,7 +422,12 @@ def build_receipt(args: argparse.Namespace) -> dict[str, Any]: "disk_before": before, "protected_container_image_count": len(protected_ids), "dangling_image_total_count": len(dangling_images), - "dangling_image_removal_plan": summarize_images(removal_candidates), + "dangling_image_removal_plan": summarize_images(dangling_removal_candidates), + "tagged_image_removal_plan": summarize_images(tagged_removal_candidates), + "eligible_image_removal_plan": summarize_images( + eligible_removal_candidates + ), + "total_image_removal_plan": summarize_images(removal_candidates), "builder_cache_command": builder_prune_command(args)[1:] if args.include_builder_cache else None, "removed_image_ids": [], "builder_cache_cleanup_executed": False, @@ -292,6 +452,15 @@ def main() -> int: if args.keep_dangling_newest < 0: print("keep-dangling-newest must be >= 0", file=sys.stderr) return 2 + if args.tagged_min_age_hours <= 0: + print("tagged-min-age-hours must be > 0", file=sys.stderr) + return 2 + if args.keep_tagged_newest_per_repository < 1: + print("keep-tagged-newest-per-repository must be >= 1", file=sys.stderr) + return 2 + if args.max_image_removals < 1: + print("max-image-removals must be >= 1", file=sys.stderr) + return 2 receipt = build_receipt(args) text = json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True) + "\n" if args.output: diff --git a/scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py b/scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py index e1e67d655..9027b0a77 100644 --- a/scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py +++ b/scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py @@ -6,7 +6,6 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from types import SimpleNamespace - ROOT = Path(__file__).resolve().parents[3] SCRIPT = ROOT / "scripts" / "ops" / "docker-disk-pressure-retention-cleanup.py" spec = importlib.util.spec_from_file_location("docker_disk_pressure_retention_cleanup", SCRIPT) @@ -14,6 +13,7 @@ module = importlib.util.module_from_spec(spec) assert spec and spec.loader sys.modules[spec.name] = module spec.loader.exec_module(module) +UTC = timezone.utc # noqa: UP017 - mirrors supported Python 3.9 runtime. def image(image_id: str, created_at: datetime, tags: tuple[str, ...] = ()): @@ -26,7 +26,7 @@ def image(image_id: str, created_at: datetime, tags: tuple[str, ...] = ()): def test_select_dangling_images_keeps_newest_and_protects_running_images() -> None: - now = datetime(2026, 7, 1, 12, tzinfo=timezone.utc) + now = datetime(2026, 7, 1, 12, tzinfo=UTC) images = [ image("sha256:running", now - timedelta(hours=72)), image("oldest", now - timedelta(hours=72)), @@ -73,7 +73,7 @@ def test_parse_docker_datetime_accepts_nanosecond_fraction() -> None: def test_summary_never_reports_volumes_or_container_cleanup_boundary() -> None: - now = datetime(2026, 7, 1, 12, tzinfo=timezone.utc) + now = datetime(2026, 7, 1, 12, tzinfo=UTC) selected = module.select_dangling_image_removals( [image("old", now - timedelta(days=3))], set(), @@ -87,6 +87,88 @@ def test_summary_never_reports_volumes_or_container_cleanup_boundary() -> None: assert summary["sample_image_ids"] == ["old"] +def test_tagged_cleanup_preserves_running_and_repository_rollback_window() -> None: + now = datetime(2026, 7, 14, 12, tzinfo=UTC) + images = [ + image("running", now - timedelta(days=30), ("registry:5000/app:running",)), + image("latest", now - timedelta(days=1), ("registry:5000/app:latest",)), + image("rollback-1", now - timedelta(days=2), ("registry:5000/app:sha-1",)), + image("rollback-2", now - timedelta(days=3), ("registry:5000/app:sha-2",)), + image("expired", now - timedelta(days=14), ("registry:5000/app:sha-old",)), + image("other-repo", now - timedelta(days=14), ("registry:5000/other:only",)), + ] + + selected = module.select_tagged_image_removals( + images, + {"running"}, + now=now, + min_age_hours=7 * 24, + keep_newest_per_repository=3, + ) + + assert [item.image_id for item in selected] == ["expired"] + + +def test_multi_repository_image_is_retained_when_any_repository_needs_it() -> None: + now = datetime(2026, 7, 14, 12, tzinfo=UTC) + shared = image( + "shared", + now - timedelta(days=30), + ("registry:5000/app:old", "registry:5000/critical:latest"), + ) + selected = module.select_tagged_image_removals( + [shared, image("app-new", now - timedelta(days=1), ("registry:5000/app:new",))], + set(), + now=now, + min_age_hours=7 * 24, + keep_newest_per_repository=1, + ) + + assert selected == [] + + +def test_repository_parser_preserves_registry_port() -> None: + assert module.repository_from_tag("registry:5000/team/app:sha") == "registry:5000/team/app" + + +def test_single_apply_image_removal_is_count_bounded() -> None: + now = datetime(2026, 7, 14, 12, tzinfo=UTC) + candidates = [ + image(f"image-{index}", now - timedelta(days=index + 1)) + for index in range(5) + ] + + selected = module.limit_image_removals(candidates, max_removals=2) + + assert [item.image_id for item in selected] == ["image-4", "image-3"] + + +def test_remove_images_batches_tagged_references(monkeypatch) -> None: + calls: list[list[str]] = [] + + def fake_docker(args: list[str], _docker_bin: str): + calls.append(args) + return SimpleNamespace(stdout="", stderr="", returncode=0) + + monkeypatch.setattr(module, "docker", fake_docker) + now = datetime(2026, 7, 14, 12, tzinfo=UTC) + + removed = module.remove_images( + [ + image("dangling", now), + image("tagged-a", now, ("repo-a:old", "repo-b:old")), + image("tagged-b", now, ("repo-c:old",)), + ], + "docker", + ) + + assert removed == ["dangling", "tagged-a", "tagged-b"] + assert calls == [ + ["image", "rm", "dangling"], + ["image", "rm", "repo-a:old", "repo-b:old", "repo-c:old"], + ] + + def test_cli_exposes_builder_cache_only_flag() -> None: help_text = module.run_command(["python3", str(SCRIPT), "--help"]).stdout From 1431abba546f195a2d67b6f99b8ae5c46ee78184 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 23:16:07 +0800 Subject: [PATCH 3/7] fix(agent99): recover host188 edge services --- agent99-bootstrap.ps1 | 10 +- agent99-contract-check.ps1 | 19 +- agent99-control-plane.ps1 | 205 ++++++++++++- agent99-deploy.ps1 | 46 ++- agent99.config.99.example.json | 23 +- agent99.config.example.json | 23 +- docs/runbooks/FULL-STACK-COLD-START-SOP.md | 16 +- .../ansible/inventory/group_vars/host_188.yml | 4 +- infra/ansible/playbooks/188-ai-web.yml | 79 +++-- .../full-stack-cold-start-baseline.yml | 20 +- scripts/ops/ansible-validate.sh | 1 + .../ai-log-triage-auto-recovery.sh | 4 +- .../cold-start-textfile-exporter.sh | 20 +- .../full-stack-cold-start-check.sh | 62 +++- .../host188-edge-services-recover.sh | 288 ++++++++++++++++++ .../test_cold_start_monitor_bounded_probes.py | 25 ++ ..._host188_edge_service_recovery_contract.py | 79 +++++ .../verify-cold-start-monitor-deploy.sh | 7 +- 18 files changed, 874 insertions(+), 57 deletions(-) create mode 100755 scripts/reboot-recovery/host188-edge-services-recover.sh create mode 100644 scripts/reboot-recovery/tests/test_host188_edge_service_recovery_contract.py diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 4325a64ba..1ad66b9e6 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -309,7 +309,15 @@ if (-not (Test-Path $configPath)) { "preferJumpHost": true, "directHosts": ["192.168.0.110", "192.168.0.112"] }, - "publicUrls": ["awoooi.wooo.work", "stock.wooo.work", "2026fifa.wooo.work", "gitea.wooo.work", "harbor.wooo.work"], + "edgeServiceRecovery": { + "enabled": true, + "host": "192.168.0.188", + "scriptPath": "/home/ollama/bin/agent99-host188-edge-services-recover.sh", + "checkTimeoutSeconds": 45, + "applyTimeoutSeconds": 900, + "requiredServices": ["n8n", "open-webui"] + }, + "publicUrls": ["https://awoooi.wooo.work", "https://stock.wooo.work", "https://2026fifa.wooo.work", "https://gitea.wooo.work", "https://harbor.wooo.work", "https://n8n.wooo.work", "https://ollama.wooo.work"], "sreAlertRelay": { "enabled": true, "prefix": "http://+:8787/agent99/sre-alert/", diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 79c481480..da8ccdcc4 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -56,7 +56,11 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js @($host111External[0].macAddresses).Count -ge 1 -and 22 -in @($host111External[0].verifyTcpPorts) ) - $hasRoutes = @($config.publicUrls).Count -ge 5 + $hasRoutes = [bool]( + @($config.publicUrls).Count -ge 7 -and + "https://n8n.wooo.work" -in @($config.publicUrls) -and + "https://ollama.wooo.work" -in @($config.publicUrls) + ) $hasSlo = [bool]($config.recoverySlo -and [int]$config.recoverySlo.targetMinutes -eq 10) $hasIdentity = [bool]($config.sshIdentityFile -and $config.sshIdentityFile -match "agent99_ed25519$") $hasCanonicalHost111User = [bool]( @@ -72,6 +76,14 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js $config.recoveryCoordinator.controllerHost -eq "192.168.0.110" -and @($config.recoveryCoordinator.requiredHostAliases).Count -eq 7 ) + $hasHost188EdgeRecovery = [bool]( + $config.edgeServiceRecovery -and + $config.edgeServiceRecovery.enabled -eq $true -and + $config.edgeServiceRecovery.host -eq "192.168.0.188" -and + $config.edgeServiceRecovery.scriptPath -eq "/home/ollama/bin/agent99-host188-edge-services-recover.sh" -and + "n8n" -in @($config.edgeServiceRecovery.requiredServices) -and + "open-webui" -in @($config.edgeServiceRecovery.requiredServices) + ) $hasCompletionCallback = [bool]( $config.completionCallback -and $config.completionCallback.enabled -eq $true -and @@ -85,8 +97,8 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js [string]$_.vmx -eq "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" }).Count -eq 1) } else { $true } - $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasCanonicalHost111User -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasCompletionCallback -and $hasCanonicalHost112Vmx - Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity canonicalHost111User=$hasCanonicalHost111User jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" + $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasCanonicalHost111User -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasHost188EdgeRecovery -and $hasCompletionCallback -and $hasCanonicalHost112Vmx + Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity canonicalHost111User=$hasCanonicalHost111User jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator host188EdgeRecovery=$hasHost188EdgeRecovery completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" } catch { Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)" } @@ -116,6 +128,7 @@ Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("manager Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route" Add-Check "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf') -and $bootstrap.Contains('Get-AgentHost112CanonicalVm $PolicyPath') -and $bootstrap.Contains('Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))')) "Host112 VM identity is normalized from one staged policy row and fails closed when absent" Add-Check "recovery:host112_timeout_reserve" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120")) "Host112 timeout chain has executor, queue and client reserves" +Add-Check "recovery:host188_edge_services" ($control.Contains("Invoke-AgentHost188EdgeRecovery") -and $control.Contains('New-AgentRecoveryPhase "host188_edge_services"') -and $control.Contains('New-AgentOutcomeCheck "host188_edge_services_ready"') -and $control.Contains("container_local_upstream_and_public_https") -and $deployer.Contains('name = "canonical_public_route_coverage"')) "Host188 n8n and Open WebUI use a pinned controlled apply and independent public-route verifier" Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present" Add-Check "callback:durable_completion" ($control.Contains('schema_version = "agent99_completion_callback_v1"') -and $control.Contains("durable_readback") -and $control.Contains("completion-callbacks\pending")) "Agent99 completion callbacks are queued until durable AWOOOI/AwoooP readback" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 7fc0ab43f..be8f0c8bc 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -4450,6 +4450,7 @@ function Get-AgentOutcomeContract { $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) $aiOk = [bool](@($data.aiServices | Where-Object { -not $_.ok }).Count -eq 0) $host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified) + $host188EdgeOk = [bool]($data.host188EdgeRecovery -and $data.host188EdgeRecovery.verified) $evidenceReconcileOnly = [bool]( $data.PSObject.Properties["reconcileOnly"] -and $data.reconcileOnly -is [bool] -and @@ -4493,6 +4494,7 @@ function Get-AgentOutcomeContract { $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk $checks += New-AgentOutcomeCheck "ai_services_healthy" $aiOk $false $checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ',')) + $checks += New-AgentOutcomeCheck "host188_edge_services_ready" $host188EdgeOk $true $([string]$data.host188EdgeRecovery.terminal) if ($RequireReconcileOnly) { $checks += New-AgentOutcomeCheck "reconcile_only_evidence_bound" $evidenceReceiptBound $true "expected=$ExpectedReconcileEvidenceId" $checks += New-AgentOutcomeCheck "reconcile_only_no_runtime_write" $reconcileNoWrite $true @@ -4503,7 +4505,7 @@ function Get-AgentOutcomeContract { $checks += New-AgentOutcomeCheck "unexpected_reconcile_only_evidence" $false $true $reconcileNoWrite = $false } - $modeVerifierPassed = [bool]($hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $reconcileNoWrite) + $modeVerifierPassed = [bool]($hostsOk -and $host112Ok -and $host188EdgeOk -and $harborOk -and $awoooOk -and $publicOk -and $reconcileNoWrite) } "Recover" { $expectedVms = @($Config.vms).Count @@ -4521,6 +4523,7 @@ function Get-AgentOutcomeContract { $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) $coordinatorOk = [bool]($data.recoveryCoordinator -and $data.recoveryCoordinator.verified) $host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified) + $host188EdgeOk = [bool]($data.host188EdgeRecovery -and $data.host188EdgeRecovery.verified) $checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)" $checks += New-AgentOutcomeCheck "external_hosts_recovered" $externalHostsOk $true "expected=$expectedExternalHosts actual=$($externalHostRows.Count)" $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" @@ -4528,8 +4531,9 @@ function Get-AgentOutcomeContract { $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk $checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ',')) + $checks += New-AgentOutcomeCheck "host188_edge_services_ready" $host188EdgeOk $true $([string]$data.host188EdgeRecovery.terminal) $checks += New-AgentOutcomeCheck "full_sop_coordinator_verified" $coordinatorOk $true $([string](@($data.recoveryCoordinator.failedChecks) -join ',')) - $modeVerifierPassed = [bool]($vmsOk -and $externalHostsOk -and $hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk) + $modeVerifierPassed = [bool]($vmsOk -and $externalHostsOk -and $hostsOk -and $host112Ok -and $host188EdgeOk -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk) } "StartVMs" { $expectedVms = @($Config.vms).Count @@ -6945,6 +6949,180 @@ function Invoke-AgentExternalHostRecovery { $results } +function Get-AgentHost188EdgeRecoveryConfig { + $configured = if ($Config.PSObject.Properties["edgeServiceRecovery"]) { $Config.edgeServiceRecovery } else { $null } + [pscustomobject]@{ + enabled = [bool](-not $configured -or -not $configured.PSObject.Properties["enabled"] -or (Convert-AgentBool $configured.enabled)) + host = if ($configured -and $configured.PSObject.Properties["host"]) { [string]$configured.host } else { "192.168.0.188" } + scriptPath = if ($configured -and $configured.PSObject.Properties["scriptPath"]) { [string]$configured.scriptPath } else { "/home/ollama/bin/agent99-host188-edge-services-recover.sh" } + checkTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["checkTimeoutSeconds"]) { [math]::Max(15, [math]::Min(120, [int]$configured.checkTimeoutSeconds)) } else { 45 } + applyTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["applyTimeoutSeconds"]) { [math]::Max(120, [math]::Min(1200, [int]$configured.applyTimeoutSeconds)) } else { 900 } + requiredServices = if ($configured -and $configured.PSObject.Properties["requiredServices"]) { @($configured.requiredServices | ForEach-Object { [string]$_ }) } else { @("n8n", "open-webui") } + } +} + +function Convert-AgentHost188EdgeReceipt { + param([object]$Transport) + + if (-not $Transport) { + return [pscustomobject]@{ + receiptPresent = $false + parseError = "transport_missing" + overallHealthy = $false + runtimeWritePerformed = $false + services = @() + } + } + + $jsonLines = @(([string]$Transport.output -split "`r?`n") | Where-Object { + $line = $_.Trim() + $line.StartsWith("{") -and $line.EndsWith("}") + }) + if ($jsonLines.Count -eq 0) { + return [pscustomobject]@{ + receiptPresent = $false + parseError = "json_receipt_missing" + overallHealthy = $false + runtimeWritePerformed = $false + services = @() + } + } + + try { + $receipt = $jsonLines[-1] | ConvertFrom-Json + $receipt | Add-Member -NotePropertyName receiptPresent -NotePropertyValue $true -Force + $receipt | Add-Member -NotePropertyName parseError -NotePropertyValue "" -Force + return $receipt + } catch { + return [pscustomobject]@{ + receiptPresent = $false + parseError = "json_receipt_invalid" + overallHealthy = $false + runtimeWritePerformed = $false + services = @() + } + } +} + +function Get-AgentSshReceiptSummary { + param([object]$Transport) + if (-not $Transport) { return $null } + [pscustomobject]@{ + ok = [bool]$Transport.ok + exitCode = [int]$Transport.exitCode + elapsedMs = $Transport.elapsedMs + route = if ($Transport.PSObject.Properties["route"]) { [string]$Transport.route } else { "" } + transportSerialized = [bool]($Transport.PSObject.Properties["transportSerialized"] -and $Transport.transportSerialized) + transportLockWaitMs = if ($Transport.PSObject.Properties["transportLockWaitMs"]) { $Transport.transportLockWaitMs } else { $null } + } +} + +function Invoke-AgentHost188EdgeRecovery { + param( + [string]$RequestedRunId, + [string]$RequestedTraceId, + [string]$RequestedWorkItemId, + [string]$FallbackRunId, + [switch]$Apply + ) + + $settings = Get-AgentHost188EdgeRecoveryConfig + $canonicalHost = "192.168.0.188" + $canonicalScript = "/home/ollama/bin/agent99-host188-edge-services-recover.sh" + $contractReady = [bool]( + $settings.enabled -and + $settings.host -eq $canonicalHost -and + $settings.scriptPath -eq $canonicalScript -and + "n8n" -in $settings.requiredServices -and + "open-webui" -in $settings.requiredServices + ) + if (-not $contractReady) { + return [pscustomobject]@{ + schemaVersion = "agent99_host188_edge_recovery_v1" + enabled = [bool]$settings.enabled + contractReady = $false + controlledApply = [bool]$Apply + applyAttempted = $false + runtimeWritePerformed = $false + changed = $false + verified = $false + terminal = "canonical_contract_invalid" + before = $null + applyReceipt = $null + after = $null + verifier = "container_local_upstream_and_public_https" + } + } + + $safeRunId = if ($RequestedRunId -match '^[A-Za-z0-9._:-]+$') { $RequestedRunId } elseif ($FallbackRunId -match '^[A-Za-z0-9._:-]+$') { $FallbackRunId } else { "agent99-edge-$([guid]::NewGuid().ToString('N'))" } + $safeTraceId = if ($RequestedTraceId -match '^[A-Za-z0-9._:-]+$') { $RequestedTraceId } else { "trace-$safeRunId" } + $safeWorkItemId = if ($RequestedWorkItemId -match '^[A-Za-z0-9._:-]+$') { $RequestedWorkItemId } else { "AIA-P0-006-EDGE-502-188" } + $quotedScript = Quote-ShSingle $settings.scriptPath + $beforeTransport = Invoke-HostSshText $settings.host "$quotedScript --check" $settings.checkTimeoutSeconds 1 + $before = Convert-AgentHost188EdgeReceipt $beforeTransport + $beforeHealthy = [bool]($before.receiptPresent -and (Convert-AgentBool $before.overallHealthy)) + $applyTransport = $null + $applyReceipt = $null + $applyAttempted = $false + $afterTransport = $beforeTransport + $after = $before + $terminal = if ($beforeHealthy) { "idempotent_already_verified_healthy" } else { "controlled_apply_required" } + + if (-not $beforeHealthy -and $Apply) { + $applyAttempted = $true + $applyCommand = "$quotedScript --apply --trace-id $(Quote-ShSingle $safeTraceId) --run-id $(Quote-ShSingle $safeRunId) --work-item-id $(Quote-ShSingle $safeWorkItemId)" + Write-AgentLog "CONTROLLED_APPLY host188_edge_services run=$safeRunId workItem=$safeWorkItemId" + $applyTransport = Invoke-HostSshText $settings.host $applyCommand $settings.applyTimeoutSeconds 1 + $applyReceipt = Convert-AgentHost188EdgeReceipt $applyTransport + $afterTransport = Invoke-HostSshText $settings.host "$quotedScript --check" $settings.checkTimeoutSeconds 2 + $after = Convert-AgentHost188EdgeReceipt $afterTransport + $afterHealthy = [bool]($after.receiptPresent -and (Convert-AgentBool $after.overallHealthy)) + if ($afterHealthy) { + $terminal = "deployed_verified" + } elseif (-not $applyTransport.ok) { + $terminal = "apply_transport_failed" + } else { + $terminal = "post_verifier_failed_snapshot_preserved" + } + } + + $verified = [bool]($after.receiptPresent -and (Convert-AgentBool $after.overallHealthy)) + $runtimeWritePerformed = [bool]( + $applyAttempted -and + $applyReceipt -and + $applyReceipt.PSObject.Properties["runtimeWritePerformed"] -and + (Convert-AgentBool $applyReceipt.runtimeWritePerformed) + ) + $result = [pscustomobject]@{ + schemaVersion = "agent99_host188_edge_recovery_v1" + enabled = $true + contractReady = $true + targetHost = $settings.host + requiredServices = @($settings.requiredServices) + controlledApply = [bool]$Apply + applyAttempted = $applyAttempted + runtimeWritePerformed = $runtimeWritePerformed + changed = [bool](-not $beforeHealthy -and $verified -and $runtimeWritePerformed) + verified = $verified + terminal = $terminal + beforeTransport = Get-AgentSshReceiptSummary $beforeTransport + before = $before + applyTransport = Get-AgentSshReceiptSummary $applyTransport + applyReceipt = $applyReceipt + afterTransport = Get-AgentSshReceiptSummary $afterTransport + after = $after + rollback = "run_scoped_snapshots_preserved_break_glass_restore_only" + verifier = "container_local_upstream_and_public_https" + prohibitedActions = @("host_reboot", "vm_power_change", "firewall_change", "database_content_read", "secret_read", "external_registry_direct_pull") + } + if ($result.changed) { + Record-AgentEvent "host188_edge_services_recovered" "info" "188 的 n8n 與 Open WebUI 已由 Agent99 受控恢復，並通過容器、本機 upstream、公開 HTTPS verifier。" $result -Alert + } elseif ($applyAttempted -and -not $verified) { + Record-AgentEvent "host188_edge_services_recovery_failed" "critical" "188 的 n8n 或 Open WebUI 仍未通過 verifier；快照已保留，未執行主機重啟或資料還原。" $result -Alert + } + $result +} + function Test-HostReachability { $results = @() foreach ($hostIp in $Config.hosts) { @@ -7350,6 +7528,7 @@ $hostResults = @() $externalHostRecovery = @() $requiredExternalHostFailures = @() $host112Recovery = $null +$host188EdgeRecovery = $null $harbor = $null $awooo = $null $public = @() @@ -7456,6 +7635,22 @@ if ($Mode -in @("Recover", "AwoooRepair") -and $awooo) { } } +if ($Mode -in @("Status", "Recover")) { + $host188EdgeRecovery = Invoke-AgentHost188EdgeRecovery ` + -RequestedRunId $AutomationRunId ` + -RequestedTraceId $TraceId ` + -RequestedWorkItemId $WorkItemId ` + -FallbackRunId $recoveryRunId ` + -Apply:($Mode -eq "Recover" -and $ControlledApply) + if ($Mode -eq "Recover") { + $edgeStatus = if ($host188EdgeRecovery.verified) { "ok" } else { "failed" } + $edgeEvidence = if ($host188EdgeRecovery.after -and $host188EdgeRecovery.after.PSObject.Properties["receiptPath"]) { [string]$host188EdgeRecovery.after.receiptPath } else { "" } + $recoveryPhases += New-AgentRecoveryPhase "host188_edge_services" $edgeStatus (((Get-Date) - $recoveryStartedAt).TotalSeconds) $edgeEvidence + } elseif (-not $ReconcileOnly -and -not $host188EdgeRecovery.verified -and -not $recoveryTrigger) { + $recoveryTrigger = Start-AgentRecoveryFromObservation "host188_edge_services_not_ready" $host188EdgeRecovery.before + } +} + if ($Mode -in @("Status", "Recover", "PublicSmoke")) { $public = Test-PublicRoutes if ($Mode -eq "Recover") { @@ -7541,7 +7736,7 @@ $summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCrit if ($Mode -eq "Recover") { $elapsedSeconds = [math]::Round(((Get-Date) - $recoveryStartedAt).TotalSeconds, 1) $unreachableCount = @($hostResults | Where-Object { -not $_.ping }).Count - $builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $requiredExternalHostFailures.Count -eq 0 -and $host112Recovery -and $host112Recovery.verified -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) + $builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $requiredExternalHostFailures.Count -eq 0 -and $host112Recovery -and $host112Recovery.verified -and $host188EdgeRecovery -and $host188EdgeRecovery.verified -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) $coordinatorVerified = [bool]($recoveryCoordinator -and $recoveryCoordinator.verified) $requiresFreshWindow = [bool]($recoveryCoordinator -and $recoveryCoordinator.requireFreshRebootWindow) $rebootSloClaimed = [bool]($requiresFreshWindow -and $recoveryCoordinator.rebootSloClaimed) @@ -7590,7 +7785,8 @@ $result = [pscustomobject]@{ reconcileQueueId = $ReconcileQueueId runtimeWritePerformed = [bool]( ($recoveryTrigger -and $recoveryTrigger.queued) -or - ($host112Recovery -and $host112Recovery.runtimeWritePerformed) + ($host112Recovery -and $host112Recovery.runtimeWritePerformed) -or + ($host188EdgeRecovery -and $host188EdgeRecovery.runtimeWritePerformed) ) sshProcessGuard = $sshProcessGuard bootState = $bootState @@ -7601,6 +7797,7 @@ $result = [pscustomobject]@{ externalHostRecovery = $externalHostRecovery hosts = $hostResults host112Recovery = $host112Recovery + host188EdgeRecovery = $host188EdgeRecovery harbor = $harbor awoooi = $awooo public = $public diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index c286130fb..f336da167 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -456,6 +456,37 @@ try { Add-DefaultProperty $config.recoveryCoordinator "maxArtifactAgeSeconds" 600 Add-DefaultProperty $config.recoveryCoordinator "artifactClockSkewSeconds" 90 Add-DefaultProperty $config.recoveryCoordinator "requiredHostAliases" @("99", "110", "111", "112", "120", "121", "188") + Add-DefaultProperty $config "edgeServiceRecovery" ([pscustomobject]@{}) + Add-DefaultProperty $config.edgeServiceRecovery "enabled" $true + Add-DefaultProperty $config.edgeServiceRecovery "host" "192.168.0.188" + Add-DefaultProperty $config.edgeServiceRecovery "scriptPath" "/home/ollama/bin/agent99-host188-edge-services-recover.sh" + Add-DefaultProperty $config.edgeServiceRecovery "checkTimeoutSeconds" 45 + Add-DefaultProperty $config.edgeServiceRecovery "applyTimeoutSeconds" 900 + Add-DefaultProperty $config.edgeServiceRecovery "requiredServices" @("n8n", "open-webui") + Add-DefaultProperty $config "publicUrls" @() + $previousPublicUrls = @($config.publicUrls | ForEach-Object { [string]$_ } | Where-Object { $_ }) + $canonicalPublicUrls = @($previousPublicUrls | ForEach-Object { + $url = $_.Trim() + if ($url -match '^https?://') { $url } else { "https://$url" } + }) + $requiredPublicUrls = @( + "https://awoooi.wooo.work", + "https://stock.wooo.work", + "https://2026fifa.wooo.work", + "https://gitea.wooo.work", + "https://harbor.wooo.work", + "https://n8n.wooo.work", + "https://ollama.wooo.work" + ) + $canonicalPublicUrls = @($canonicalPublicUrls + $requiredPublicUrls | Select-Object -Unique) + if (($previousPublicUrls -join ',') -cne ($canonicalPublicUrls -join ',')) { + Set-AgentProperty $config "publicUrls" $canonicalPublicUrls + $script:ConfigMigrations += [pscustomobject]@{ + name = "canonical_public_route_coverage" + from = ($previousPublicUrls -join ",") + to = ($canonicalPublicUrls -join ",") + } + } Add-DefaultProperty $config "selfHealth" ([pscustomobject]@{}) Add-DefaultProperty $config.selfHealth "requiredHosts" @("192.168.0.110", "192.168.0.111", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188") $selfHealthHosts = @($config.selfHealth.requiredHosts | ForEach-Object { [string]$_ } | Where-Object { $_ }) @@ -509,6 +540,16 @@ try { $persistedHost112Vmx = if ($persistedHost112Vm.Count -eq 1) { [string]$persistedHost112Vm[0].vmx } else { "" } $persistedHosts = @($persistedConfig.hosts | ForEach-Object { [string]$_ }) $persistedSelfHealthHosts = @($persistedConfig.selfHealth.requiredHosts | ForEach-Object { [string]$_ }) + $persistedPublicUrls = @($persistedConfig.publicUrls | ForEach-Object { [string]$_ }) + $persistedEdgeServices = if ($persistedConfig.PSObject.Properties["edgeServiceRecovery"]) { $persistedConfig.edgeServiceRecovery } else { $null } + $persistedEdgeServicesReady = [bool]( + $persistedEdgeServices -and + [bool]$persistedEdgeServices.enabled -and + [string]$persistedEdgeServices.host -eq "192.168.0.188" -and + [string]$persistedEdgeServices.scriptPath -eq "/home/ollama/bin/agent99-host188-edge-services-recover.sh" -and + "n8n" -in @($persistedEdgeServices.requiredServices | ForEach-Object { [string]$_ }) -and + "open-webui" -in @($persistedEdgeServices.requiredServices | ForEach-Object { [string]$_ }) + ) $persistedHost111External = @($persistedConfig.externalHosts | Where-Object { [string]$_.host -eq "192.168.0.111" }) $host111ExternalReady = [bool]( $persistedHost111External.Count -eq 1 -and @@ -524,6 +565,9 @@ try { "192.168.0.112" -notin $persistedDirectHosts -or "192.168.0.111" -notin $persistedHosts -or "192.168.0.111" -notin $persistedSelfHealthHosts -or + "https://n8n.wooo.work" -notin $persistedPublicUrls -or + "https://ollama.wooo.work" -notin $persistedPublicUrls -or + -not $persistedEdgeServicesReady -or -not $host111ExternalReady -or $persistedHost112Vm.Count -ne 1 -or [string]$persistedHost112Vm[0].name -ne [string]$canonicalHost112Vm.name -or @@ -531,7 +575,7 @@ try { $persistedHost112Vmx -ne $canonicalHost112Vmx -or -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) ) { - throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed." + throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed. Host188 edge recovery config post-verifier also failed." } $manifestRows = @() diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 48b481e49..7474b5fb3 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -126,12 +126,25 @@ "188" ] }, + "edgeServiceRecovery": { + "enabled": true, + "host": "192.168.0.188", + "scriptPath": "/home/ollama/bin/agent99-host188-edge-services-recover.sh", + "checkTimeoutSeconds": 45, + "applyTimeoutSeconds": 900, + "requiredServices": [ + "n8n", + "open-webui" + ] + }, "publicUrls": [ - "awoooi.wooo.work", - "stock.wooo.work", - "2026fifa.wooo.work", - "gitea.wooo.work", - "harbor.wooo.work" + "https://awoooi.wooo.work", + "https://stock.wooo.work", + "https://2026fifa.wooo.work", + "https://gitea.wooo.work", + "https://harbor.wooo.work", + "https://n8n.wooo.work", + "https://ollama.wooo.work" ], "aiServices": [ { diff --git a/agent99.config.example.json b/agent99.config.example.json index d970d69fe..744ab7088 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -120,12 +120,25 @@ "188" ] }, + "edgeServiceRecovery": { + "enabled": true, + "host": "192.168.0.188", + "scriptPath": "/home/ollama/bin/agent99-host188-edge-services-recover.sh", + "checkTimeoutSeconds": 45, + "applyTimeoutSeconds": 900, + "requiredServices": [ + "n8n", + "open-webui" + ] + }, "publicUrls": [ - "awoooi.wooo.work", - "stock.wooo.work", - "2026fifa.wooo.work", - "gitea.wooo.work", - "harbor.wooo.work" + "https://awoooi.wooo.work", + "https://stock.wooo.work", + "https://2026fifa.wooo.work", + "https://gitea.wooo.work", + "https://harbor.wooo.work", + "https://n8n.wooo.work", + "https://ollama.wooo.work" ], "aiServices": [ { diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 4ac798562..647a0d9b7 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.134 +> Version: v1.136 > Last updated: 2026-07-14 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -10,6 +10,20 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天，必須先重跑 live check，再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 +### v1.136 2026-07-14 Host112 正式納入 cold-start P0 release gate + +`full-stack-cold-start-check.sh` 舊版仍顯示 `112 Kali is intentionally skipped`，與 Windows99 五台 required VM、Agent99 `host112_guest_readiness` phase 及全主機 10 分鐘 SLO 不一致。現行 baseline 已把 112 從 excluded host 移入 included host；P0 network 固定檢查 ping / TCP 22，並透過 `kali@192.168.0.112` 執行 canonical `/usr/local/sbin/awoooi-host112-guest-readiness --check`。release gate 必須同時驗證 readback schema、systemd running、guest/console/services/recovery timer、Wazuh 1514/1515/55000 independent verifier 與 `runtime_write_performed=0`。任何一項失敗均維持 BLOCKED；不得因其他四台 Linux host 或 public routes 為 200 而宣稱全主機恢復。110 textfile exporter scope 固定為 `110_112_120_121_188`，並分開輸出 112 host unreachable 與 guest readiness failed blocker series；`verify-cold-start-monitor-deploy.sh` 必須驗證 source/live hash、scope 與 blocker series parity，避免 checker 已納管但 Prometheus 仍漏掉 112。 + +### v1.135 2026-07-14 Host188 n8n / Open WebUI 502 自動恢復 + +本輪全資產 public smoke 發現 `https://n8n.wooo.work` 與 `https://ollama.wooo.work` 均為 raw `502`。188 的 Nginx 正常，但 upstream `127.0.0.1:5678` / `127.0.0.1:3010` 沒有可用服務；Ansible 宣稱 `/opt/n8n`、`/opt/open-webui` 可直接 `docker compose up`，live 目錄卻不存在。n8n 的實際資料與 base compose 在 `/home/ollama/n8n`，Open WebUI 只剩 `open-webui` volume；兩個本機映像都已遺失。這類事件必須分類為「服務部署資產漂移」，不能只重啟 Nginx、不能把 502 當 warmup，也不能從外部 registry 拉 mutable `latest`。 + +- 受控恢復固定使用內部 Harbor immutable digest：n8n `sha256:dbe732489e5b8941aa89ca71b320ccb7f80319fcccd0f2c9e05f561562d804e5`；Open WebUI `sha256:a673af6e7fab29822a0d32d71184d3f0277347a693633c86372643590c921915`。apply 前先建立 run-scoped n8n data / Open WebUI volume 快照，任何資料 restore 仍是 break-glass，不得自動執行。 +- 188 Docker 啟用 `userns-remap`，`dockremap` 起始 UID/GID 為 `100000`。n8n image 的 `node` UID/GID `1000` 必須映射成 host `101000:101000`；直接加 `userns_mode: host` 會讓 image layer 內 `/home/node/.cache` 再次 EACCES，因此正式修法是收斂 bind data ownership、維持預設 user namespace。 +- canonical executor 是 `scripts/reboot-recovery/host188-edge-services-recover.sh`，live path `/home/ollama/bin/agent99-host188-edge-services-recover.sh`。`--check` 僅讀 container/image/restart/health、本機 upstream 與 local-Nginx HTTPS；`--apply` 才可拉固定 Harbor digest、建立快照、原子寫入 managed compose、修正 n8n remap ownership、啟動服務，再跑獨立 `--check`。 +- live apply `trace-agent99-edge-services-20260714 / run-agent99-edge-services-20260714-02 / AIA-P0-006-EDGE-502-188` 終態為 `deployed_verified`：n8n 與 Open WebUI 均 `running`、restart count `0`、local/public HTTP `200`、config managed、image pinned；receipt 在 `/home/ollama/agent99-runtime-recovery/run-agent99-edge-services-20260714-02/receipt.json`。後續 no-write check 為 exit `0`、`runtimeWritePerformed=false`，兩份 compose mtime 不變。 +- Agent99 `Status` 必須偵測此 lane 並排入單一 Recover；Recover 固定執行 Detect -> check -> bounded apply -> independent verifier -> callback/TG/KM。`full-stack-cold-start-check.sh` 與 AI log triage public routes 必須同時包含 n8n / Open WebUI，兩者任一未通過時不得宣稱所有網站恢復。 + ### v1.134 2026-07-14 Agent99 canonical host readback closure 本輪釐清 111 的 `perf_readback_failed` 不是 CPU 壓力，而是 Agent99 沿用預設 `wooo`，但 110 到 111 的 canonical restricted identity 是 `ooo`。部署合約固定 `sshUsers["192.168.0.111"]="ooo"`，舊 live config 必須由 `host111_canonical_ssh_user` migration 寫入後重新讀回；帳號、transport 或四項指標任一缺失時，只能標示 readback degraded，不得產生 CPU/記憶體/磁碟故障結論或執行降載。 diff --git a/infra/ansible/inventory/group_vars/host_188.yml b/infra/ansible/inventory/group_vars/host_188.yml index 0bc802d51..63d96345b 100644 --- a/infra/ansible/inventory/group_vars/host_188.yml +++ b/infra/ansible/inventory/group_vars/host_188.yml @@ -19,10 +19,10 @@ docker_compose_services: dir: /opt/minio expected_port: 9000 n8n: - dir: /opt/n8n + dir: /home/ollama/n8n expected_port: 5678 open_webui: - dir: /opt/open-webui + dir: /home/ollama/open-webui expected_port: 3010 docker_registry: dir: /opt/docker-registry diff --git a/infra/ansible/playbooks/188-ai-web.yml b/infra/ansible/playbooks/188-ai-web.yml index 1091574b8..24da34aae 100644 --- a/infra/ansible/playbooks/188-ai-web.yml +++ b/infra/ansible/playbooks/188-ai-web.yml @@ -177,35 +177,72 @@ # ======================================================================== # n8n / open-webui (Sprint A 新啟動) # ======================================================================== - - name: "N8N | 確認容器運作中" + - name: "Edge services | 確認受控恢復腳本目錄存在" + ansible.builtin.file: + path: /home/ollama/bin + state: directory + owner: ollama + group: ollama + mode: "0755" + tags: + - n8n + - open_webui + - edge_services + + - name: "Edge services | 安裝 Agent99 受控恢復腳本" + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../../scripts/reboot-recovery/host188-edge-services-recover.sh" + dest: /home/ollama/bin/agent99-host188-edge-services-recover.sh + owner: ollama + group: ollama + mode: "0755" + tags: + - n8n + - open_webui + - edge_services + + - name: "Edge services | 讀取 n8n 與 Open WebUI 前置狀態" + become: false ansible.builtin.command: - cmd: "docker ps --filter name=n8n --filter status=running --format '{{ '{{' }}.Names{{ '}}' }}'" - register: n8n_status + cmd: /home/ollama/bin/agent99-host188-edge-services-recover.sh --check + register: edge_services_before changed_when: false - tags: n8n + failed_when: edge_services_before.rc not in [0, 2] + check_mode: false + tags: + - n8n + - open_webui + - edge_services - - name: "N8N | 若停止則啟動" + - name: "Edge services | 執行有界恢復與獨立 verifier" + become: false ansible.builtin.command: - cmd: "docker compose up -d" - chdir: /opt/n8n + cmd: >- + /home/ollama/bin/agent99-host188-edge-services-recover.sh + --apply + --trace-id ansible-host188-edge-{{ ansible_date_time.epoch }} + --run-id ansible-host188-edge-{{ ansible_date_time.epoch }} + --work-item-id AIA-P0-006-EDGE-502-188 + when: + - not ansible_check_mode + - edge_services_before.rc == 2 changed_when: true - when: n8n_status.stdout == "" - tags: n8n + tags: + - n8n + - open_webui + - edge_services - - name: "Open-webui | 確認容器運作中" + - name: "Edge services | 驗證容器、本機 upstream 與公開 HTTPS" + become: false ansible.builtin.command: - cmd: "docker ps --filter name=open-webui --filter status=running --format '{{ '{{' }}.Names{{ '}}' }}'" - register: openwebui_status + cmd: /home/ollama/bin/agent99-host188-edge-services-recover.sh --check + register: edge_services_after changed_when: false - tags: open_webui - - - name: "Open-webui | 若停止則啟動" - ansible.builtin.command: - cmd: "docker compose up -d" - chdir: /opt/open-webui - changed_when: true - when: openwebui_status.stdout == "" - tags: open_webui + when: not ansible_check_mode or edge_services_before.rc == 0 + tags: + - n8n + - open_webui + - edge_services # ======================================================================== # Nginx 狀態確認 diff --git a/ops/reboot-recovery/full-stack-cold-start-baseline.yml b/ops/reboot-recovery/full-stack-cold-start-baseline.yml index e17f42d2f..d0782f042 100644 --- a/ops/reboot-recovery/full-stack-cold-start-baseline.yml +++ b/ops/reboot-recovery/full-stack-cold-start-baseline.yml @@ -1,12 +1,12 @@ -version: 2026-05-06.v1 +version: 2026-07-14.v2 scope: included_hosts: "110": "DevOps, registry, observability, Sentry, runners" + "112": "Kali security, Wazuh, guest readiness and bounded recovery" "120": "K3s control plane and VIP" "121": "K3s peer node and DR drill cron" "188": "Data, AI, web, momo, SignOz, public nginx gateway" - excluded_hosts: - "112": "Kali security host; recorded but not part of cold-start release gate" + excluded_hosts: {} principles: - recover_dependency_chain_before_workloads @@ -19,8 +19,8 @@ phases: - id: P0-NETWORK order: 0 gates: - - ping_110_120_121_188 - - ssh_port_110_120_121_188 + - ping_110_112_120_121_188 + - ssh_port_110_112_120_121_188 - arp_evidence_or_monitor_mode_fallback - id: P0-188-DATA @@ -54,6 +54,16 @@ phases: - cadvisor_image_v0_47_0 - cadvisor_cpu_cap_0_3 + - id: P0-112-SECURITY + order: 25 + gates: + - canonical_guest_readback_schema + - systemd_running + - graphical_console_and_vmware_tools_ready + - wazuh_stack_and_independent_ports_ready + - guest_recovery_timer_ready + - check_mode_performs_no_runtime_write + - id: P1-K3S order: 30 gates: diff --git a/scripts/ops/ansible-validate.sh b/scripts/ops/ansible-validate.sh index 5427b5c13..c84cd6d90 100755 --- a/scripts/ops/ansible-validate.sh +++ b/scripts/ops/ansible-validate.sh @@ -31,6 +31,7 @@ python3 ops/runner/guard-gitea-runner-pressure.py --root "$ROOT_DIR" echo "== Shell 語法 ==" bash -n \ scripts/reboot-recovery/full-stack-cold-start-check.sh \ + scripts/reboot-recovery/host188-edge-services-recover.sh \ scripts/reboot-recovery/full-stack-recovery-scorecard.sh \ scripts/reboot-recovery/dr-offsite-operator-checklist.sh \ scripts/reboot-recovery/wait-dr-offsite-ready.sh \ diff --git a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh index a330ef026..6a535ecf5 100755 --- a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh +++ b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh @@ -231,7 +231,9 @@ probe_public_routes() { "https://stock.wooo.work/api/v1/system/freshness" \ "https://gitea.wooo.work/" \ "https://registry.wooo.work/v2/" \ - "https://harbor.wooo.work/" + "https://harbor.wooo.work/" \ + "https://n8n.wooo.work/" \ + "https://ollama.wooo.work/" do code="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)" [ -n "$code" ] || code="000" diff --git a/scripts/reboot-recovery/cold-start-textfile-exporter.sh b/scripts/reboot-recovery/cold-start-textfile-exporter.sh index 2798f8775..2477b41e9 100755 --- a/scripts/reboot-recovery/cold-start-textfile-exporter.sh +++ b/scripts/reboot-recovery/cold-start-textfile-exporter.sh @@ -14,7 +14,7 @@ CHECK_TIMEOUT_SECONDS="${CHECK_TIMEOUT_SECONDS:-240}" CHECK_WATCH_INTERVAL_SECONDS="${CHECK_WATCH_INTERVAL_SECONDS:-10}" CHECK_WATCH_MAX_ATTEMPTS="${CHECK_WATCH_MAX_ATTEMPTS:-3}" HOST_LABEL="${AIOPS_HOST_LABEL:-110}" -SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_120_121_188}" +SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_112_120_121_188}" LOCK_FILE="${LOCK_FILE:-/tmp/awoooi-cold-start-textfile-exporter.lock}" escape_label() { @@ -39,6 +39,8 @@ write_metric_file() { local public_route_tls_blocker="${15}" local host_120_unreachable_blocker="${16}" local backup_health_blocker="${17}" + local host_112_unreachable_blocker="${18}" + local host_112_guest_not_ready_blocker="${19}" local host scope host=$(escape_label "$HOST_LABEL") scope=$(escape_label "$SCOPE_LABEL") @@ -80,6 +82,8 @@ awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="k3s_node_fi awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="public_route_tls_failure",target="public_https"} $public_route_tls_blocker awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="host_unreachable",target="120"} $host_120_unreachable_blocker awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="backup_health_blocked",target="110"} $backup_health_blocker +awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="host_unreachable",target="112"} $host_112_unreachable_blocker +awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="guest_readiness_failed",target="112"} $host_112_guest_not_ready_blocker METRICS } @@ -102,7 +106,7 @@ if [ ! -x "$CHECK_SCRIPT" ]; then tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX") last_green=$(cat "$state_file" 2>/dev/null || echo 0) printf 'CHECK_SCRIPT not executable: %s\n' "$CHECK_SCRIPT" >"$log_file" - write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green" 0 0 0 0 + write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green" 0 0 0 0 0 0 chmod 0644 "$tmp_metric" mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME" exit 0 @@ -131,6 +135,8 @@ k3s_node_fs_blocker=0 public_route_tls_blocker=0 host_120_unreachable_blocker=0 backup_health_blocker=0 +host_112_unreachable_blocker=0 +host_112_guest_not_ready_blocker=0 if [ -n "$summary_line" ]; then monitor_up=1 @@ -166,6 +172,14 @@ if grep -Eq 'BLOCKED 110 backup health has stale expected jobs' "$log_file"; the backup_health_blocker=1 fi +if grep -Eq 'BLOCKED (ping 192\.168\.0\.112|ssh port 192\.168\.0\.112:22|112 canonical guest readback unavailable)' "$log_file"; then + host_112_unreachable_blocker=1 +fi + +if grep -Eq 'BLOCKED 112 (canonical guest readback schema missing|systemd not running|guest readiness failed|console or required services not ready|Wazuh independent verifier failed|guest recovery timer not ready|cold-start probe write boundary violated)' "$log_file"; then + host_112_guest_not_ready_blocker=1 +fi + end_ts=$(date +%s) if [ "$green" -eq 1 ]; then printf '%s\n' "$end_ts" >"$state_file" @@ -173,6 +187,6 @@ fi last_green=$(cat "$state_file" 2>/dev/null || echo 0) tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX") -write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green" "$k3s_node_fs_blocker" "$public_route_tls_blocker" "$host_120_unreachable_blocker" "$backup_health_blocker" +write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green" "$k3s_node_fs_blocker" "$public_route_tls_blocker" "$host_120_unreachable_blocker" "$backup_health_blocker" "$host_112_unreachable_blocker" "$host_112_guest_not_ready_blocker" chmod 0644 "$tmp_metric" mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME" diff --git a/scripts/reboot-recovery/full-stack-cold-start-check.sh b/scripts/reboot-recovery/full-stack-cold-start-check.sh index be248399f..acaa22d5d 100755 --- a/scripts/reboot-recovery/full-stack-cold-start-check.sh +++ b/scripts/reboot-recovery/full-stack-cold-start-check.sh @@ -246,6 +246,13 @@ registry_code_ready() { [ "$code" = "200" ] || [ "$code" = "401" ] } +readback_value() { + local text="$1" key="$2" rest + rest="${text#*${key}=}" + [ "$rest" != "$text" ] || return 1 + printf '%s' "${rest%% *}" +} + probe_tcp() { local host="$1" local port="$2" @@ -254,11 +261,11 @@ probe_tcp() { print_neighbor_rows() { if command -v arp >/dev/null 2>&1; then - arp -an | grep -E '192\.168\.0\.(110|120|121|188)' + arp -an | grep -E '192\.168\.0\.(110|112|120|121|188)' return $? fi if command -v ip >/dev/null 2>&1; then - ip neigh show | grep -E '192\.168\.0\.(110|120|121|188)' + ip neigh show | grep -E '192\.168\.0\.(110|112|120|121|188)' return $? fi return 1 @@ -267,14 +274,14 @@ print_neighbor_rows() { print_header() { echo "AWOOOI full-stack cold-start check" date '+%Y-%m-%d %H:%M:%S %Z' - echo "Scope: 110 / 120 / 121 / 188. 112 Kali is intentionally skipped." + echo "Scope: 110 / 112 / 120 / 121 / 188." echo "Baseline: ops/reboot-recovery/full-stack-cold-start-baseline.yml" } check_network() { log_section "P0-NETWORK" local host - for host in 110 120 121 188; do + for host in 110 112 120 121 188; do if ping -c 1 -W 2 "192.168.0.$host" >/dev/null 2>&1; then ok "ping 192.168.0.$host" else @@ -297,6 +304,50 @@ check_network() { fi } +check_112() { + log_section "P0-112-SECURITY" + local out schema systemd_state guest_ready console_ready services_ready + local recovery_timer_ready manager_verifier_ready runtime_write_performed + if ! out=$(host_cmd "kali@192.168.0.112" \ + '/usr/local/sbin/awoooi-host112-guest-readiness --check' 2>&1); then + fail "112 canonical guest readback unavailable" + echo "$out" + return + fi + + schema="$(readback_value "$out" schema_version || true)" + systemd_state="$(readback_value "$out" systemd_state || true)" + guest_ready="$(readback_value "$out" guest_ready || true)" + console_ready="$(readback_value "$out" console_ready || true)" + services_ready="$(readback_value "$out" services_ready || true)" + recovery_timer_ready="$(readback_value "$out" recovery_timer_ready || true)" + manager_verifier_ready="$(readback_value "$out" manager_independent_verifier_ready || true)" + runtime_write_performed="$(readback_value "$out" runtime_write_performed || true)" + echo "HOST112_READBACK schema=${schema:-missing} systemd=${systemd_state:-missing} guest=${guest_ready:-missing} console=${console_ready:-missing} services=${services_ready:-missing} timer=${recovery_timer_ready:-missing} manager_verifier=${manager_verifier_ready:-missing} runtime_write=${runtime_write_performed:-missing}" + + [ "$schema" = "host112_guest_recovery_v2" ] \ + && ok "112 canonical guest readback schema verified" \ + || fail "112 canonical guest readback schema missing" + [ "$systemd_state" = "running" ] \ + && ok "112 systemd running" \ + || fail "112 systemd not running" + [ "$guest_ready" = "1" ] \ + && ok "112 guest readiness verified" \ + || fail "112 guest readiness failed" + [ "$console_ready" = "1" ] && [ "$services_ready" = "1" ] \ + && ok "112 console and required services ready" \ + || fail "112 console or required services not ready" + [ "$manager_verifier_ready" = "1" ] \ + && ok "112 Wazuh independent verifier ready" \ + || fail "112 Wazuh independent verifier failed" + [ "$recovery_timer_ready" = "1" ] \ + && ok "112 guest recovery timer ready" \ + || fail "112 guest recovery timer not ready" + [ "$runtime_write_performed" = "0" ] \ + && ok "112 cold-start probe remained read-only" \ + || fail "112 cold-start probe write boundary violated" +} + check_188() { log_section "P0-188-DATA" local out @@ -660,6 +711,8 @@ check_public_routes() { "langfuse|https://langfuse.wooo.work/" "bitan|https://bitan.wooo.work/" "aiops|https://aiops.wooo.work/" + "n8n|https://n8n.wooo.work/" + "open_webui|https://ollama.wooo.work/" ) for item in "${routes[@]}"; do @@ -1067,6 +1120,7 @@ fi print_header check_network +check_112 check_188 check_110 check_k3s diff --git a/scripts/reboot-recovery/host188-edge-services-recover.sh b/scripts/reboot-recovery/host188-edge-services-recover.sh new file mode 100755 index 000000000..77e839ca9 --- /dev/null +++ b/scripts/reboot-recovery/host188-edge-services-recover.sh @@ -0,0 +1,288 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="check" +TRACE_ID="" +RUN_ID="" +WORK_ITEM_ID="" + +HOST_ID="192.168.0.188" +N8N_IMAGE="harbor.wooo.work/dockerhub-cache/n8nio/n8n@sha256:dbe732489e5b8941aa89ca71b320ccb7f80319fcccd0f2c9e05f561562d804e5" +N8N_IMAGE_ID="sha256:19c1ad26c0c285fddd7fc1300ff670b7977afebf52a21de4dfb2a5a2b2a75e2b" +OPEN_WEBUI_IMAGE="harbor.wooo.work/ghcr-cache/open-webui/open-webui@sha256:a673af6e7fab29822a0d32d71184d3f0277347a693633c86372643590c921915" +OPEN_WEBUI_IMAGE_ID="sha256:5bddf2d93bc307b99c1fde8b8d0c2dc3c563e0e24fc3eebb6488c539fa55d080" +BACKUP_IMAGE="harbor.wooo.work/dockerhub-cache/library/alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc" +N8N_DIR="/home/ollama/n8n" +N8N_OVERRIDE="${N8N_DIR}/docker-compose.agent99.yml" +OPEN_WEBUI_DIR="/home/ollama/open-webui" +OPEN_WEBUI_COMPOSE="${OPEN_WEBUI_DIR}/docker-compose.yml" +RECEIPT_ROOT="/home/ollama/agent99-runtime-recovery" + +usage() { + printf 'usage: %s --check | --apply --trace-id ID --run-id ID --work-item-id ID\n' "$0" >&2 + exit 64 +} + +valid_id() { + [[ "$1" =~ ^[A-Za-z0-9._:-]+$ ]] +} + +while (($#)); do + case "$1" in + --check) + MODE="check" + shift + ;; + --apply) + MODE="apply" + shift + ;; + --trace-id) + (($# >= 2)) || usage + TRACE_ID="$2" + shift 2 + ;; + --run-id) + (($# >= 2)) || usage + RUN_ID="$2" + shift 2 + ;; + --work-item-id) + (($# >= 2)) || usage + WORK_ITEM_ID="$2" + shift 2 + ;; + *) + usage + ;; + esac +done + +if [[ "$MODE" == "apply" ]]; then + [[ -n "$TRACE_ID" && -n "$RUN_ID" && -n "$WORK_ITEM_ID" ]] || usage + valid_id "$TRACE_ID" && valid_id "$RUN_ID" && valid_id "$WORK_ITEM_ID" || usage +fi + +bool_json() { + if [[ "$1" == "true" ]]; then printf 'true'; else printf 'false'; fi +} + +http_status() { + local url="$1" + local resolve_arg="${2:-}" + local status + if [[ -n "$resolve_arg" ]]; then + status="$(curl -k -sS -L -o /dev/null -w '%{http_code}' --resolve "$resolve_arg" --max-time 12 "$url" 2>/dev/null || true)" + else + status="$(curl -sS -L -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)" + fi + [[ "$status" =~ ^[0-9]{3}$ ]] || status="000" + printf '%s' "$status" +} + +non_5xx() { + local status="$1" + [[ "$status" =~ ^[0-9]{3}$ ]] && ((10#$status >= 200 && 10#$status < 500)) +} + +container_value() { + local name="$1" + local format="$2" + docker inspect "$name" --format "$format" 2>/dev/null || true +} + +collect_state() { + local runtime_write="${1:-false}" + local terminal="${2:-observed}" + local receipt_path="${3:-}" + + local n8n_status n8n_image n8n_restart n8n_restarts n8n_local n8n_public + local webui_status webui_health webui_image webui_restart webui_restarts webui_local webui_public + local n8n_config_ok=false webui_config_ok=false n8n_ok=false webui_ok=false overall=false + + n8n_status="$(container_value n8n '{{.State.Status}}')" + n8n_image="$(container_value n8n '{{.Image}}')" + n8n_restart="$(container_value n8n '{{.HostConfig.RestartPolicy.Name}}')" + n8n_restarts="$(container_value n8n '{{.RestartCount}}')" + [[ "$n8n_restarts" =~ ^[0-9]+$ ]] || n8n_restarts=0 + n8n_local="$(http_status 'http://127.0.0.1:5678/')" + n8n_public="$(http_status 'https://n8n.wooo.work/' 'n8n.wooo.work:443:127.0.0.1')" + + webui_status="$(container_value open-webui '{{.State.Status}}')" + webui_health="$(container_value open-webui '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}')" + webui_image="$(container_value open-webui '{{.Image}}')" + webui_restart="$(container_value open-webui '{{.HostConfig.RestartPolicy.Name}}')" + webui_restarts="$(container_value open-webui '{{.RestartCount}}')" + [[ "$webui_restarts" =~ ^[0-9]+$ ]] || webui_restarts=0 + webui_local="$(http_status 'http://127.0.0.1:3010/')" + webui_public="$(http_status 'https://ollama.wooo.work/' 'ollama.wooo.work:443:127.0.0.1')" + + if [[ -f "$N8N_OVERRIDE" ]] && grep -Fqx " image: ${N8N_IMAGE}" "$N8N_OVERRIDE"; then + n8n_config_ok=true + fi + if [[ -f "$OPEN_WEBUI_COMPOSE" ]] && grep -Fqx " image: ${OPEN_WEBUI_IMAGE}" "$OPEN_WEBUI_COMPOSE"; then + webui_config_ok=true + fi + + if [[ "$n8n_config_ok" == true && "$n8n_status" == running && "$n8n_image" == "$N8N_IMAGE_ID" && "$n8n_restart" == always ]] && non_5xx "$n8n_local" && non_5xx "$n8n_public"; then + n8n_ok=true + fi + if [[ "$webui_config_ok" == true && "$webui_status" == running && "$webui_health" == healthy && "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" && "$webui_restart" == always ]] && non_5xx "$webui_local" && non_5xx "$webui_public"; then + webui_ok=true + fi + if [[ "$n8n_ok" == true && "$webui_ok" == true ]]; then + overall=true + fi + + printf '{"schemaVersion":"agent99_host188_edge_services_v1","mode":"%s","traceId":"%s","runId":"%s","workItemId":"%s","host":"%s","overallHealthy":%s,"runtimeWritePerformed":%s,"secretValueRead":false,"rawDataRead":false,"terminal":"%s","receiptPath":"%s","services":[{"name":"n8n","publicUrl":"https://n8n.wooo.work","containerStatus":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s},{"name":"open-webui","publicUrl":"https://ollama.wooo.work","containerStatus":"%s","health":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s}],"rollback":{"automatic":false,"snapshotPreserved":true,"procedure":"stop_changed_containers_then_restore_run_scoped_snapshot_after_owner_break_glass"},"prohibitedActions":["host_reboot","vm_power_change","firewall_change","database_content_read","secret_read","external_registry_direct_pull"]}\n' \ + "$MODE" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$HOST_ID" \ + "$(bool_json "$overall")" "$(bool_json "$runtime_write")" "$terminal" "$receipt_path" \ + "$n8n_status" "$(bool_json "$([[ "$n8n_image" == "$N8N_IMAGE_ID" ]] && echo true || echo false)")" "$n8n_restart" "$n8n_restarts" "$n8n_local" "$n8n_public" "$(bool_json "$n8n_config_ok")" "$(bool_json "$n8n_ok")" \ + "$webui_status" "$webui_health" "$(bool_json "$([[ "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" ]] && echo true || echo false)")" "$webui_restart" "$webui_restarts" "$webui_local" "$webui_public" "$(bool_json "$webui_config_ok")" "$(bool_json "$webui_ok")" + + [[ "$overall" == true ]] +} + +if [[ "$MODE" == "check" ]]; then + if collect_state false observed ""; then + exit 0 + fi + exit 2 +fi + +exec 9>/tmp/agent99-host188-edge-services.lock +if ! flock -w 30 9; then + collect_state false single_flight_lock_timeout "" || true + exit 75 +fi + +RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}" +OPERATION_LOG="${RECEIPT_DIR}/operation.log" +RECEIPT_PATH="${RECEIPT_DIR}/receipt.json" + +emit_apply_failure() { + local exit_code="$1" + local result + trap - ERR + set +e + result="$(collect_state true "apply_failed_exit_${exit_code}" "$RECEIPT_PATH" || true)" + printf '%s\n' "$result" | tee "$RECEIPT_PATH" + chmod 600 "$RECEIPT_PATH" 2>/dev/null || true + exit "$exit_code" +} + +mkdir -p "$RECEIPT_DIR" "$N8N_DIR" "$OPEN_WEBUI_DIR" +chmod 700 "$RECEIPT_DIR" +: >"$OPERATION_LOG" +chmod 600 "$OPERATION_LOG" +trap 'emit_apply_failure $?' ERR + +for image in "$BACKUP_IMAGE" "$N8N_IMAGE" "$OPEN_WEBUI_IMAGE"; do + timeout 1200 docker pull "$image" >>"$OPERATION_LOG" 2>&1 +done + +snapshot_with_helper() { + local source_path="$1" + local archive_name="$2" + if [[ -s "${RECEIPT_DIR}/${archive_name}" ]]; then + return 0 + fi + docker run --rm --pull never --userns=host \ + -v "${source_path}:/data:ro" -v "${RECEIPT_DIR}:/backup" \ + "$BACKUP_IMAGE" sh -c "umask 077; tar czf /backup/${archive_name} -C /data ." \ + >>"$OPERATION_LOG" 2>&1 +} + +[[ -d "${N8N_DIR}/data" && -f "${N8N_DIR}/docker-compose.yml" ]] || { + collect_state false n8n_base_assets_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true + exit 3 +} + +if ! docker volume inspect open-webui >/dev/null 2>&1; then + docker volume create open-webui >>"$OPERATION_LOG" +fi + +snapshot_with_helper "${N8N_DIR}/data" "n8n-data-before.tgz" +docker run --rm --pull never --userns=host \ + -v open-webui:/data:ro -v "${RECEIPT_DIR}:/backup" \ + "$BACKUP_IMAGE" sh -c 'umask 077; tar czf /backup/open-webui-volume-before.tgz -C /data .' \ + >>"$OPERATION_LOG" 2>&1 +docker run --rm --pull never --userns=host -v "${RECEIPT_DIR}:/backup" "$BACKUP_IMAGE" \ + chown -R "$(id -u):$(id -g)" /backup >>"$OPERATION_LOG" 2>&1 +chmod 600 "${RECEIPT_DIR}/n8n-data-before.tgz" "${RECEIPT_DIR}/open-webui-volume-before.tgz" + +tmp_n8n="$(mktemp "${N8N_OVERRIDE}.XXXXXX")" +cat >"$tmp_n8n" <"$tmp_webui" </dev/null || true)" +remap_gid="$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subgid 2>/dev/null || true)" +if [[ "$remap_uid" =~ ^[0-9]+$ && "$remap_gid" =~ ^[0-9]+$ ]]; then + node_uid=$((remap_uid + 1000)) + node_gid=$((remap_gid + 1000)) + docker run --rm --pull never --userns=host -v "${N8N_DIR}/data:/data" "$BACKUP_IMAGE" \ + chown -R "${node_uid}:${node_gid}" /data >>"$OPERATION_LOG" 2>&1 +else + collect_state true docker_userns_mapping_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true + exit 4 +fi + +docker compose --project-directory "$N8N_DIR" \ + -f "${N8N_DIR}/docker-compose.yml" -f "$N8N_OVERRIDE" \ + up -d --pull never >>"$OPERATION_LOG" 2>&1 +webui_project="$(docker inspect open-webui --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || true)" +if [[ -n "$(docker ps -aq --filter name='^/open-webui$')" && "$webui_project" != "open-webui" ]]; then + docker rm -f open-webui >>"$OPERATION_LOG" 2>&1 +fi +docker compose --project-directory "$OPEN_WEBUI_DIR" -f "$OPEN_WEBUI_COMPOSE" \ + up -d --pull never >>"$OPERATION_LOG" 2>&1 + +deadline=$((SECONDS + 360)) +while ((SECONDS < deadline)); do + if collect_state true verifying "$RECEIPT_PATH" >/dev/null; then + break + fi + sleep 5 +done + +if result="$(collect_state true deployed_verified "$RECEIPT_PATH")"; then + trap - ERR + printf '%s\n' "$result" | tee "$RECEIPT_PATH" + chmod 600 "$RECEIPT_PATH" + exit 0 +fi + +result="$(collect_state true verifier_failed_snapshot_preserved "$RECEIPT_PATH" || true)" +trap - ERR +printf '%s\n' "$result" | tee "$RECEIPT_PATH" +chmod 600 "$RECEIPT_PATH" +exit 5 diff --git a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py index 91f6fa0ce..05270aaa4 100644 --- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py +++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py @@ -63,6 +63,31 @@ def test_full_stack_cold_start_check_bounds_ssh_probes() -> None: assert "SSH_110_RECOVERY_PACKAGE_NEXT_ACTION verify_or_preinstall_local_recovery_package_from_console_before_harbor_repair_retry" in text +def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None: + text = COLD_START_CHECK.read_text(encoding="utf-8") + baseline = (ROOT / "ops" / "reboot-recovery" / "full-stack-cold-start-baseline.yml").read_text( + encoding="utf-8" + ) + exporter = (ROOT / "scripts" / "reboot-recovery" / "cold-start-textfile-exporter.sh").read_text( + encoding="utf-8" + ) + deploy_verifier = VERIFY_DEPLOY.read_text(encoding="utf-8") + + assert "Scope: 110 / 112 / 120 / 121 / 188." in text + assert "for host in 110 112 120 121 188; do" in text + assert 'log_section "P0-112-SECURITY"' in text + assert "awoooi-host112-guest-readiness --check" in text + assert 'check_112\ncheck_188' in text + assert '"112": "Kali security, Wazuh, guest readiness and bounded recovery"' in baseline + assert "- id: P0-112-SECURITY" in baseline + assert "112 Kali is intentionally skipped" not in text + assert "not part of cold-start release gate" not in baseline + assert 'SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_112_120_121_188}"' in exporter + assert 'reason="host_unreachable",target="112"' in exporter + assert 'reason="guest_readiness_failed",target="112"' in exporter + assert 'scope="110_112_120_121_188"' in deploy_verifier + + def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None: text = COLD_START_CHECK.read_text(encoding="utf-8") diff --git a/scripts/reboot-recovery/tests/test_host188_edge_service_recovery_contract.py b/scripts/reboot-recovery/tests/test_host188_edge_service_recovery_contract.py new file mode 100644 index 000000000..6cb365c4d --- /dev/null +++ b/scripts/reboot-recovery/tests/test_host188_edge_service_recovery_contract.py @@ -0,0 +1,79 @@ +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] + + +def read(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def test_agent99_config_covers_host188_edge_services_and_public_routes() -> None: + for name in ("agent99.config.example.json", "agent99.config.99.example.json"): + config = json.loads(read(name)) + edge = config["edgeServiceRecovery"] + + assert edge["enabled"] is True + assert edge["host"] == "192.168.0.188" + assert edge["scriptPath"] == "/home/ollama/bin/agent99-host188-edge-services-recover.sh" + assert set(edge["requiredServices"]) == {"n8n", "open-webui"} + assert "https://n8n.wooo.work" in config["publicUrls"] + assert "https://ollama.wooo.work" in config["publicUrls"] + + +def test_host188_recovery_uses_only_internal_pinned_images() -> None: + script = read("scripts/reboot-recovery/host188-edge-services-recover.sh") + + assert "harbor.wooo.work/dockerhub-cache/n8nio/n8n@sha256:" in script + assert "harbor.wooo.work/ghcr-cache/open-webui/open-webui@sha256:" in script + assert "harbor.wooo.work/dockerhub-cache/library/alpine@sha256:" in script + assert "ghcr.io/" not in script + assert "docker.n8n.io/" not in script + assert ":latest" not in script + assert "external_registry_direct_pull" in script + + +def test_host188_check_mode_is_read_only_and_apply_is_bounded() -> None: + script = read("scripts/reboot-recovery/host188-edge-services-recover.sh") + check_block = script.split('if [[ "$MODE" == "check" ]]', 1)[1].split( + "exec 9>", 1 + )[0] + + for forbidden in ( + "docker pull", + "docker run", + "docker compose", + "mkdir", + "mv ", + "chown", + "chmod", + "tee ", + ): + assert forbidden not in check_block + + assert "flock -w 30" in script + assert "snapshot_with_helper" in script + assert "--pull never" in script + assert '"secretValueRead":false' in script + assert '"rawDataRead":false' in script + assert '"automatic":false' in script + assert "host_reboot" in script + assert "vm_power_change" in script + + +def test_agent99_recover_binds_check_apply_verify_and_completion() -> None: + control = read("agent99-control-plane.ps1") + deploy = read("agent99-deploy.ps1") + playbook = read("infra/ansible/playbooks/188-ai-web.yml") + + assert "function Invoke-AgentHost188EdgeRecovery" in control + assert 'New-AgentRecoveryPhase "host188_edge_services"' in control + assert 'New-AgentOutcomeCheck "host188_edge_services_ready"' in control + assert "$host188EdgeRecovery.runtimeWritePerformed" in control + assert "$host188EdgeRecovery.verified" in control + assert "container_local_upstream_and_public_https" in control + assert 'name = "canonical_public_route_coverage"' in deploy + assert "agent99-host188-edge-services-recover.sh --check" in playbook + assert "--work-item-id AIA-P0-006-EDGE-502-188" in playbook + assert "not ansible_check_mode" in playbook diff --git a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh index d55face23..c968df054 100755 --- a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh +++ b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh @@ -129,10 +129,15 @@ require_remote_pattern \ "110 deployed check script carries SSH host-key policy" require_remote_pattern \ - 'awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188",mode="read_only"} 1' \ + 'awoooi_cold_start_monitor_up{host="110",scope="110_112_120_121_188",mode="read_only"} 1' \ "/home/wooo/node_exporter_textfiles/cold_start_recovery.prom" \ "110 cold-start monitor produced parseable metrics" +require_remote_pattern \ + 'awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="guest_readiness_failed",target="112"}' \ + "/home/wooo/node_exporter_textfiles/cold_start_recovery.prom" \ + "110 cold-start monitor exports host112 readiness blocker series" + report_runtime_state report_cold_start_alerts From 180e708444e4c9e3d30b8a52746d3dedf00eccd8 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 19:14:11 +0800 Subject: [PATCH 4/7] feat(telegram): enforce canonical product routing --- agent99-control-plane.ps1 | 58 +- apps/api/src/api/v1/gitea_webhook.py | 14 +- apps/api/src/api/v1/sentry_webhook.py | 25 +- apps/api/src/api/v1/signoz_webhook.py | 56 +- apps/api/src/api/v1/telegram.py | 15 +- apps/api/src/api/v1/webhooks.py | 75 +- apps/api/src/jobs/ai_slo_watchdog_job.py | 38 +- apps/api/src/jobs/capacity_forecaster_job.py | 30 +- apps/api/src/jobs/compliance_scanner_job.py | 20 +- apps/api/src/jobs/coverage_evaluator_job.py | 21 +- apps/api/src/jobs/hermes_rule_quality_job.py | 32 +- ...agent_report_truth_actionability_review.py | 16 +- apps/api/src/services/ai_rate_limiter.py | 127 +- apps/api/src/services/ai_router.py | 8 +- apps/api/src/services/approval_execution.py | 111 +- apps/api/src/services/channel_hub.py | 52 +- .../converged_alert_recurrence_notifier.py | 39 +- apps/api/src/services/decision_manager.py | 234 +- apps/api/src/services/drift_adopt_service.py | 5 +- apps/api/src/services/drift_remediator.py | 7 +- .../services/emergency_escalation_service.py | 3 - apps/api/src/services/failover_alerter.py | 171 +- apps/api/src/services/failure_watcher.py | 35 +- .../api/src/services/gitea_webhook_service.py | 15 +- .../src/services/image_analysis_service.py | 7 +- apps/api/src/services/k3s_monitor_service.py | 19 +- apps/api/src/services/notification_matrix.py | 800 ++++++- .../src/services/notifications/telegram.py | 55 +- .../src/services/post_execution_verifier.py | 38 +- .../src/services/report_generation_service.py | 150 +- apps/api/src/services/runbook_generator.py | 12 +- apps/api/src/services/telegram_gateway.py | 1940 ++++++++++++++--- .../api/src/services/weekly_report_service.py | 25 +- .../tests/test_alert_converged_recurrence.py | 9 +- .../tests/test_ansible_verified_closure.py | 16 + .../test_channel_hub_grouped_alert_events.py | 83 +- apps/api/tests/test_failover_alerter.py | 17 +- .../test_notification_matrix_group_cutover.py | 51 +- .../tests/test_report_generation_service.py | 13 +- .../tests/test_telegram_button_consistency.py | 5 +- .../test_telegram_callback_context_routing.py | 721 ++++++ ...est_telegram_canonical_routing_registry.py | 494 +++++ .../test_telegram_canonical_sender_gate.py | 521 +++++ .../test_telegram_delivery_truth_callers.py | 399 ++++ ...test_telegram_delivery_truth_callers_v2.py | 394 ++++ ...test_telegram_delivery_truth_gateway_v2.py | 460 ++++ .../tests/test_telegram_message_templates.py | 87 +- docs/adr/ADR-093-telegram-group-migration.md | 6 +- ..._canonical_routing_registry_v1.schema.json | 309 +++ .../TELEGRAM-CANONICAL-ROUTING-INVENTORY.md | 60 + ...m-canonical-routing-registry.snapshot.json | 522 +++++ .../reboot-recovery/awoooi-telegram-notify.sh | 19 +- .../telegram-notification-egress-inventory.py | 39 +- ...notification-egress-no-new-bypass-guard.py | 8 +- 54 files changed, 7746 insertions(+), 740 deletions(-) create mode 100644 apps/api/tests/test_telegram_callback_context_routing.py create mode 100644 apps/api/tests/test_telegram_canonical_routing_registry.py create mode 100644 apps/api/tests/test_telegram_canonical_sender_gate.py create mode 100644 apps/api/tests/test_telegram_delivery_truth_callers.py create mode 100644 apps/api/tests/test_telegram_delivery_truth_callers_v2.py create mode 100644 apps/api/tests/test_telegram_delivery_truth_gateway_v2.py create mode 100644 docs/schemas/telegram_canonical_routing_registry_v1.schema.json create mode 100644 docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md create mode 100644 docs/security/telegram-canonical-routing-registry.snapshot.json diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index be8f0c8bc..bfd210e6e 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -547,6 +547,18 @@ function Send-AgentTelegramRelay { return [pscustomobject]@{ ok = $false; error = "relay_disabled" } } + # Canonical Telegram routing is owned by the AWOOOI API gateway. Agent99 + # must never borrow a container token/chat binding or call Bot API directly. + # Until a correlated gateway transport is configured this path is an + # explicit no-egress terminal, not a best-effort fallback. + return [pscustomobject]@{ + ok = $false + sent = $false + providerSendPerformed = $false + routeStatus = "blocked_no_egress" + error = "canonical_telegram_gateway_transport_required" + } + $relayHost = if ($Relay.host) { [string]$Relay.host } else { "192.168.0.110" } $container = if ($Relay.container) { [string]$Relay.container } else { "stockplatform-v2-api-1" } if ($container -notmatch "^[A-Za-z0-9_.-]+$") { @@ -618,7 +630,7 @@ if photo_path and os.path.exists(photo_path): request_data["reply_parameters"] = reply_parameters with open(photo_path, "rb") as fh: response = requests.post( - "https://api.telegram.org/bot" + token + "/sendPhoto", + "blocked://canonical-telegram-gateway-required/sendPhoto", data=request_data, files={"photo": (filename, fh, ctype)}, timeout=30, @@ -646,7 +658,7 @@ if photo_path and os.path.exists(photo_path): body.extend(fh.read()) body.extend(b"\r\n") body.extend(("--" + boundary + "--\r\n").encode()) - req = urllib.request.Request("https://api.telegram.org/bot" + token + "/sendPhoto", data=bytes(body)) + req = urllib.request.Request("blocked://canonical-telegram-gateway-required/sendPhoto", data=bytes(body)) req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary) try: response_payload = json.loads(urllib.request.urlopen(req, timeout=30).read().decode()) @@ -660,7 +672,7 @@ else: if reply_parameters: fields["reply_parameters"] = reply_parameters data = urllib.parse.urlencode(fields).encode() - response_payload = json.loads(urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read().decode()) + response_payload = json.loads(urllib.request.urlopen("blocked://canonical-telegram-gateway-required/sendMessage", data=data, timeout=15).read().decode()) emit_receipt(response_payload) print("sent_ok") ' @@ -1641,6 +1653,14 @@ function Send-AgentTelegramPhotoDirect { [string]$ReplyToMessageId = "" ) + return [pscustomobject]@{ + ok = $false + sent = $false + providerSendPerformed = $false + routeStatus = "blocked_no_egress" + error = "canonical_telegram_gateway_transport_required" + } + Add-Type -AssemblyName System.Net.Http -ErrorAction Stop $client = New-Object System.Net.Http.HttpClient $content = New-Object System.Net.Http.MultipartFormDataContent @@ -1660,7 +1680,7 @@ function Send-AgentTelegramPhotoDirect { $fileContent = [System.Net.Http.ByteArrayContent]::new($fileBytes) $fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("image/png") $content.Add($fileContent, "photo", [IO.Path]::GetFileName($PhotoPath)) - $response = $client.PostAsync("https://api.telegram.org/bot$Token/sendPhoto", $content).GetAwaiter().GetResult() + $response = $client.PostAsync("blocked://canonical-telegram-gateway-required/sendPhoto", $content).GetAwaiter().GetResult() $body = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() if (-not $response.IsSuccessStatusCode) { throw "telegram_sendPhoto_failed status=$([int]$response.StatusCode) body=$body" @@ -1940,10 +1960,13 @@ function Send-AgentTelegram { return $null } - $tokenSecret = Get-AgentEnvSecret $tokenEnvAliases - $chatSecret = Get-AgentEnvSecret $chatIdEnvAliases - $token = if ($tokenSecret) { $tokenSecret.value } else { $null } - $chatId = if ($chatSecret) { $chatSecret.value } else { $null } + # The canonical gateway transport is not wired into Agent99 yet. Keep all + # credential bindings unset so the fail-closed receipt below cannot read a + # host-local bot token/chat identifier before returning. + $tokenSecret = $null + $chatSecret = $null + $token = $null + $chatId = $null $chatIdOverride = if ($Data -and $Data.PSObject.Properties["replyChatId"]) { [string]$Data.replyChatId } else { "" } $card = Get-AgentIncidentCardModel $Severity $EventType $Message $Data $incidentState = Get-AgentTelegramIncidentState $card @@ -1982,6 +2005,21 @@ function Send-AgentTelegram { $script:TelegramAttempts += $attempt return $attempt } + + # Fail closed before reading a sender binding, copying an attachment, or + # attempting any provider call. Agent99 events must be routed by the + # canonical API registry/gateway with a durable receipt. + $attempt.error = "canonical_telegram_gateway_transport_required" + $attempt.sent = $false + $attempt.relay = [pscustomobject]@{ + ok = $false + providerSendPerformed = $false + routeStatus = "blocked_no_egress" + error = "canonical_telegram_gateway_transport_required" + } + $script:TelegramAttempts += $attempt + return $attempt + $text = Format-AgentTelegramText $Severity $EventType $Message $Data $caption = Format-AgentTelegramCaption $Severity $EventType $Message $Data $visualPath = Get-AgentTelegramVisualPath $Data @@ -2029,7 +2067,7 @@ function Send-AgentTelegram { disable_web_page_preview = "true" } if ($replyToMessageId) { $fallbackBody.reply_parameters = (@{ message_id = [long]$replyToMessageId } | ConvertTo-Json -Compress) } - $fallbackResult = Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body $fallbackBody + $fallbackResult = Invoke-RestMethod -Method Post -Uri "blocked://canonical-telegram-gateway-required/sendMessage" -Body $fallbackBody $attempt.sent = $true $attempt.messageId = if ($fallbackResult.result) { $fallbackResult.result.message_id } else { $null } } @@ -2040,7 +2078,7 @@ function Send-AgentTelegram { disable_web_page_preview = "true" } if ($replyToMessageId) { $textBody.reply_parameters = (@{ message_id = [long]$replyToMessageId } | ConvertTo-Json -Compress) } - $textResult = Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body $textBody + $textResult = Invoke-RestMethod -Method Post -Uri "blocked://canonical-telegram-gateway-required/sendMessage" -Body $textBody $attempt.sent = $true $attempt.messageId = if ($textResult.result) { $textResult.result.message_id } else { $null } } diff --git a/apps/api/src/api/v1/gitea_webhook.py b/apps/api/src/api/v1/gitea_webhook.py index d19c23ff5..2ff390543 100644 --- a/apps/api/src/api/v1/gitea_webhook.py +++ b/apps/api/src/api/v1/gitea_webhook.py @@ -422,10 +422,18 @@ async def _send_gitea_notification( get_telegram_gateway, # type: ignore[import] ) gateway = get_telegram_gateway() - await gateway.initialize() - await gateway.send_alert_notification(message) + result = await gateway.send_alert_notification( + message, + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) - logger.info("gitea_tg_notification_sent", dedup_key=dedup_key) + logger.info( + "gitea_tg_notification_routed", + dedup_key=dedup_key, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) except Exception as e: logger.warning("gitea_tg_notification_failed", dedup_key=dedup_key, error=str(e)) diff --git a/apps/api/src/api/v1/sentry_webhook.py b/apps/api/src/api/v1/sentry_webhook.py index d1a462e6d..25494fd6e 100644 --- a/apps/api/src/api/v1/sentry_webhook.py +++ b/apps/api/src/api/v1/sentry_webhook.py @@ -47,7 +47,10 @@ from src.services.sentry_webhook_service import ( SentrySignatureError, verify_sentry_signature, ) -from src.services.telegram_gateway import get_telegram_gateway +from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) from src.utils.timezone import now_taipei_iso logger = structlog.get_logger(__name__) @@ -648,7 +651,7 @@ async def send_sentry_telegram_alert( # 發送 Sentry 告警卡片 (含 Y/n 按鈕) # TODO(2026-04-05): Sentry 路徑無 incident_id，待 Sentry→Incident 關聯後補傳 - await telegram.send_approval_card( + delivery_result = await telegram.send_approval_card( approval_id=approval_id, risk_level="high" if level in ["fatal", "error"] else "medium", resource_name=culprit, @@ -660,13 +663,21 @@ async def send_sentry_telegram_alert( anomaly_frequency=anomaly_frequency, # 2026-04-02 ogt: 修復 ai_provider 未傳遞 → Telegram 顯示「AI 仲裁判定」而非具體模型名稱 ai_provider=analysis.analyzed_by if analysis else "", + notification_type="TYPE-3", ) - logger.info( - "sentry_telegram_sent", - approval_id=approval_id, - escalation_level=anomaly_frequency.get("escalation_level") if anomaly_frequency else None, - ) + if _telegram_send_delivery_succeeded(delivery_result): + logger.info( + "sentry_telegram_sent", + approval_id=approval_id, + escalation_level=anomaly_frequency.get("escalation_level") if anomaly_frequency else None, + ) + else: + logger.warning( + "sentry_telegram_no_send", + approval_id=approval_id, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) except Exception as e: logger.exception("sentry_telegram_failed", error=str(e)) diff --git a/apps/api/src/api/v1/signoz_webhook.py b/apps/api/src/api/v1/signoz_webhook.py index a8e5ae331..cb8394444 100644 --- a/apps/api/src/api/v1/signoz_webhook.py +++ b/apps/api/src/api/v1/signoz_webhook.py @@ -40,7 +40,10 @@ from src.services.anomaly_counter import get_anomaly_counter from src.services.approval_db import get_approval_service from src.services.channel_hub import record_external_alert_event from src.services.incident_service import get_incident_service -from src.services.telegram_gateway import get_telegram_gateway +from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) from src.utils.timezone import now_taipei_iso if TYPE_CHECKING: @@ -550,7 +553,7 @@ async def send_signoz_telegram( summary = annotations.get("summary", f"SignOz Alert: {alert_name}") description = annotations.get("description", "") - await telegram.send_approval_card( + delivery_result = await telegram.send_approval_card( approval_id=approval_id, risk_level=analysis_result.risk_level if analysis_result else ( "critical" if severity == "critical" else ( @@ -569,14 +572,23 @@ async def send_signoz_telegram( # 2026-04-02 ogt: 修復 ai_provider 未傳遞 → Telegram 顯示「AI 仲裁判定」而非具體模型名稱 ai_provider=ai_provider if ai_provider != "none" else "", incident_id=incident_id, + notification_type="TYPE-3", ) - logger.info( - "signoz_telegram_sent", - approval_id=approval_id, - alert_name=alert_name, - escalation_level=anomaly_frequency.get("escalation_level") if anomaly_frequency else None, - ) + if _telegram_send_delivery_succeeded(delivery_result): + logger.info( + "signoz_telegram_sent", + approval_id=approval_id, + alert_name=alert_name, + escalation_level=anomaly_frequency.get("escalation_level") if anomaly_frequency else None, + ) + else: + logger.warning( + "signoz_telegram_no_send", + approval_id=approval_id, + alert_name=alert_name, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) except Exception as e: logger.exception("signoz_telegram_error", error=str(e)) @@ -602,13 +614,25 @@ async def _send_log_summary_notification( from src.services.telegram_gateway import get_telegram_gateway try: + tg = get_telegram_gateway() + if not tg.canonical_destination_chat_id( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + ): + logger.info( + "log_summary_notification_no_route", + pod=pod_name, + namespace=namespace, + ) + return + svc = get_log_summary_service() summary = await svc.summarize_with_soft_timeout(pod_name, namespace) if not summary: return - tg = get_telegram_gateway() msg = ( f"🔍 Log 異常摘要\n" f"Pod: {_html.escape(pod_name)}\n" @@ -616,7 +640,19 @@ async def _send_log_summary_notification( f"{_html.escape(summary)}\n\n" f"deepseek-r1:14b | 免費本地推理" ) - await tg.send_text(msg[:4096]) + delivery_result = await tg.send_text( + msg[:4096], + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + ) + if not _telegram_send_delivery_succeeded(delivery_result): + logger.warning( + "log_summary_notification_no_send", + pod=pod_name, + namespace=namespace, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) except Exception as e: logger.warning("log_summary_notification_failed", pod=pod_name, error=str(e)) diff --git a/apps/api/src/api/v1/telegram.py b/apps/api/src/api/v1/telegram.py index 70dc95b55..f83ed62f6 100644 --- a/apps/api/src/api/v1/telegram.py +++ b/apps/api/src/api/v1/telegram.py @@ -35,7 +35,11 @@ from src.services.security_interceptor import ( NonceReplayError, UserNotWhitelistedError, ) -from src.services.telegram_gateway import TelegramGatewayError, get_telegram_gateway +from src.services.telegram_gateway import ( + TelegramGatewayError, + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) logger = get_logger("awoooi.telegram") router = APIRouter(prefix="/telegram", tags=["Telegram"]) @@ -306,6 +310,7 @@ async def telegram_webhook( await svc.download_and_analyze( chat_id=chat_id, file_id=file_id, + original_message_id=msg.get("message_id"), question=caption, ) except Exception as _img_err: @@ -353,6 +358,7 @@ async def telegram_webhook( message_id=message_id, original_text=original_text, username=username, + chat_id=message.get("chat", {}).get("id"), ) if not result.get("success"): @@ -534,11 +540,14 @@ async def test_push( suggested_action=request.suggested_action, estimated_downtime=request.estimated_downtime, incident_id=request.incident_id, + notification_type="TYPE-3", ) + delivery_succeeded = _telegram_send_delivery_succeeded(result) + return { - "ok": True, - "message": "Test push sent", + "ok": delivery_succeeded, + "message": "Test push sent" if delivery_succeeded else "Test push no-send", "telegram_response": result, } diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 8d266141e..0c8bb69e8 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -98,7 +98,11 @@ from src.services.security_interceptor import check_webhook_nonce # P0-06: nonc from src.services.signal_producer import SignalData, get_signal_producer # Phase 5: Telegram Gateway (行動戰情室) -from src.services.telegram_gateway import TelegramGatewayError, get_telegram_gateway +from src.services.telegram_gateway import ( + TelegramGatewayError, + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) # Phase 18.1.7: K8s 資源名稱正規化 已移至 alert_analyzer_service (R4 #129) from src.utils.timezone import now_taipei @@ -904,21 +908,32 @@ async def _legacy_try_auto_repair_background_disabled( if result: try: _pb_name = decision.playbook.name if decision.playbook else "unknown" - await get_telegram_gateway().mark_auto_repaired( + telegram_delivered = await get_telegram_gateway().mark_auto_repaired( approval_id=approval_id, playbook_name=_pb_name, execution_time_ms=result.execution_time_ms, success=result.success, ) - await op_log.append( - "TELEGRAM_RESULT_SENT", - incident_id=incident_id, - approval_id=approval_id, - actor="system", - action_detail="auto_repair_card_updated", - success=result.success, - context={"target_resource": target_resource, "namespace": namespace}, - ) + if telegram_delivered: + await op_log.append( + "TELEGRAM_RESULT_SENT", + incident_id=incident_id, + approval_id=approval_id, + actor="system", + action_detail="auto_repair_card_updated", + success=result.success, + context={"target_resource": target_resource, "namespace": namespace}, + ) + else: + await op_log.append( + "NOTIFICATION_CLASSIFIED", + incident_id=incident_id, + approval_id=approval_id, + actor="system", + action_detail="auto_repair_card_no_send", + success=False, + context={"target_resource": target_resource, "namespace": namespace}, + ) except Exception as tg_err: logger.warning("auto_repair_telegram_notify_failed", error=str(tg_err)) @@ -989,6 +1004,11 @@ async def _push_to_telegram_background( """ try: gateway = get_telegram_gateway() + effective_notification_type = ( + "TYPE-3" + if notification_type.strip().lower() in {"", "generic"} + else notification_type + ) # 檢查是否有設定 Bot Token if not settings.OPENCLAW_TG_BOT_TOKEN: @@ -1002,13 +1022,21 @@ async def _push_to_telegram_background( # TYPE-4D: Config Drift 使用專屬卡片 (send_drift_card) # ADR-071-F: [查看Diff][採納變更][回滾][忽略] 四鍵格式 if notification_type == "TYPE-4D": - await gateway.send_drift_card( + delivery_result = await gateway.send_drift_card( incident_id=incident_id, approval_id=approval_id, resource_name=resource_name[:50], diff_summary=diff_summary or root_cause, detected_at="", ) + if not _telegram_send_delivery_succeeded(delivery_result): + logger.warning( + "telegram_push_no_send_type4d", + approval_id=approval_id, + incident_id=incident_id, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) + return logger.info( "telegram_push_success_type4d", approval_id=approval_id, @@ -1021,7 +1049,7 @@ async def _push_to_telegram_background( # 2026-04-12 ogt: ADR-075 斷點 E 修復 — TYPE-8M Meta-System 使用專屬卡片 # alertchain_health / flywheel_health → ⚙️ META SYSTEM 卡片，不發群組 if notification_type == "TYPE-8M" or alert_category in ("alertchain_health", "flywheel_health"): - await gateway.send_meta_alert( + delivery_result = await gateway.send_meta_alert( incident_id=incident_id, approval_id=approval_id, alertname=resource_name, @@ -1030,6 +1058,14 @@ async def _push_to_telegram_background( severity_level=risk_level, system_impact=root_cause[:150], ) + if not _telegram_send_delivery_succeeded(delivery_result): + logger.warning( + "telegram_push_no_send_type8m", + approval_id=approval_id, + incident_id=incident_id, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) + return logger.info( "telegram_push_success_type8m", approval_id=approval_id, @@ -1045,7 +1081,7 @@ async def _push_to_telegram_background( if hit_count > 1: root_cause_with_count = f"[x{hit_count}] {root_cause}" - await gateway.send_approval_card( + delivery_result = await gateway.send_approval_card( approval_id=approval_id, risk_level=risk_level, resource_name=resource_name[:50], @@ -1073,7 +1109,7 @@ async def _push_to_telegram_background( incident_id=incident_id, # ADR-075 斷點 B 修復: 傳入分類以啟用動態按鈕 alert_category=alert_category, - notification_type=notification_type, + notification_type=effective_notification_type, repair_candidate_blocker_summary=repair_candidate_blocker_summary, repair_candidate_next_step=repair_candidate_next_step, repair_candidate_required_fields=repair_candidate_required_fields, @@ -1082,6 +1118,15 @@ async def _push_to_telegram_background( repair_candidate_work_item_id=repair_candidate_work_item_id, ) + if not _telegram_send_delivery_succeeded(delivery_result): + logger.warning( + "telegram_push_no_send", + approval_id=approval_id, + notification_type=effective_notification_type, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) + return + logger.info( "telegram_push_success", approval_id=approval_id, diff --git a/apps/api/src/jobs/ai_slo_watchdog_job.py b/apps/api/src/jobs/ai_slo_watchdog_job.py index 5748b6b05..e4181a65a 100644 --- a/apps/api/src/jobs/ai_slo_watchdog_job.py +++ b/apps/api/src/jobs/ai_slo_watchdog_job.py @@ -256,7 +256,7 @@ async def _check_once() -> None: dedup_key = f"watchdog:alert:{dedup_hash}" redis = get_redis() # setnx atomic — 同時多個 pod 只有第一個能 set，避免並發多發 - set_ok = await redis.set(dedup_key, "1", ex=_DEDUP_TTL_SEC, nx=True) + set_ok = await redis.set(dedup_key, "pending", ex=_DEDUP_TTL_SEC, nx=True) if not set_ok: logger.debug("ai_slo_watchdog_deduped", key=dedup_key) return @@ -281,9 +281,12 @@ async def _check_once() -> None: severity = "critical" if len(violations) >= 2 else "warning" incident_id = f"META-{now_taipei().strftime('%Y%m%d%H%M%S')}" try: - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) - await get_telegram_gateway().send_meta_alert( + delivery = await get_telegram_gateway().send_meta_alert( incident_id=incident_id, approval_id=str(uuid.uuid4()), alertname="AI 自健診異常", @@ -293,6 +296,27 @@ async def _check_once() -> None: system_impact=system_impact, probable_cause=probable_cause, ) + if not _telegram_send_delivery_succeeded(delivery): + await redis.delete(dedup_key) + logger.error( + "ai_slo_watchdog_alert_no_send", + incident_id=incident_id, + violation_count=len(violations), + delivery_status=( + delivery.get("_awooop_delivery_status", "missing_provider_ack") + if isinstance(delivery, dict) + else "missing_provider_ack" + ), + ) + return + try: + await redis.set(dedup_key, "sent", ex=_DEDUP_TTL_SEC) + except Exception as finalize_error: + logger.warning( + "ai_slo_watchdog_dedup_finalize_failed", + key=dedup_key, + error=str(finalize_error), + ) logger.warning( "ai_slo_watchdog_alert_sent", incident_id=incident_id, @@ -300,6 +324,14 @@ async def _check_once() -> None: violations=violations, ) except Exception as e: + try: + await redis.delete(dedup_key) + except Exception as release_error: + logger.warning( + "ai_slo_watchdog_dedup_release_failed", + key=dedup_key, + error=str(release_error), + ) logger.error("ai_slo_watchdog_telegram_failed", error=str(e), violations=violations) diff --git a/apps/api/src/jobs/capacity_forecaster_job.py b/apps/api/src/jobs/capacity_forecaster_job.py index 67657921a..048d23c54 100644 --- a/apps/api/src/jobs/capacity_forecaster_job.py +++ b/apps/api/src/jobs/capacity_forecaster_job.py @@ -323,12 +323,15 @@ async def _send_telegram_forecast( """ try: import html - from src.services.ai_advisory_helpers import build_ai_advisory_keyboard, is_snoozed - from src.services.telegram_gateway import get_telegram_gateway - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not target_chat_id: - return False + from src.services.ai_advisory_helpers import ( + build_ai_advisory_keyboard, + is_snoozed, + ) + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) # Snooze check: 過濾掉被人工 snooze 的 host (按「忽略 24h」後) active_risks = {} @@ -382,14 +385,15 @@ async def _send_telegram_forecast( msg = "\n".join(lines) tg = get_telegram_gateway() - await tg._send_request("sendMessage", { # type: ignore[attr-defined] - "chat_id": target_chat_id, - "text": msg, - "parse_mode": "HTML", - "disable_web_page_preview": True, - "reply_markup": keyboard, - }) - return True + result = await tg.send_canonical_message( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + text=msg, + parse_mode="HTML", + reply_markup=keyboard, + ) + return _telegram_send_delivery_succeeded(result) except Exception as e: logger.warning("capacity_forecast_telegram_failed", error=str(e)) return False diff --git a/apps/api/src/jobs/compliance_scanner_job.py b/apps/api/src/jobs/compliance_scanner_job.py index 5023254bf..30b15998b 100644 --- a/apps/api/src/jobs/compliance_scanner_job.py +++ b/apps/api/src/jobs/compliance_scanner_job.py @@ -474,17 +474,12 @@ async def _send_telegram_posture( try: import html - from src.core.config import settings from src.services.ai_advisory_helpers import ( build_ai_advisory_keyboard, is_snoozed, ) from src.services.telegram_gateway import get_telegram_gateway - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not target_chat_id: - return - # Snooze check (advisory_id 用當日 date 即可,一天只能 snooze 一次) from src.utils.timezone import now_taipei today = now_taipei().date().isoformat() @@ -532,13 +527,14 @@ async def _send_telegram_posture( include_produce_cmd=False, ) tg = get_telegram_gateway() - await tg._send_request("sendMessage", { # type: ignore[attr-defined] - "chat_id": target_chat_id, - "text": msg, - "parse_mode": "HTML", - "disable_web_page_preview": True, - "reply_markup": keyboard, - }) + await tg.send_canonical_message( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + text=msg, + parse_mode="HTML", + reply_markup=keyboard, + ) except Exception as e: logger.warning("compliance_telegram_failed", error=str(e)) diff --git a/apps/api/src/jobs/coverage_evaluator_job.py b/apps/api/src/jobs/coverage_evaluator_job.py index 74029a06b..267ee79bb 100644 --- a/apps/api/src/jobs/coverage_evaluator_job.py +++ b/apps/api/src/jobs/coverage_evaluator_job.py @@ -298,18 +298,12 @@ async def _send_telegram_gaps( """推 coverage 缺口 Telegram 摘要 + 互動按鈕 (P0 修).""" try: import html - - from src.core.config import settings from src.services.ai_advisory_helpers import ( build_ai_advisory_keyboard, is_snoozed, ) from src.services.telegram_gateway import get_telegram_gateway - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not target_chat_id: - return - # Snooze check: 以 worst_dimension 為 key worst_dim = str(analysis.get("worst_dimension", "unknown")) if await is_snoozed("coverage_gap", worst_dim): @@ -348,13 +342,14 @@ async def _send_telegram_gaps( include_produce_cmd=False, ) tg = get_telegram_gateway() - await tg._send_request("sendMessage", { # type: ignore[attr-defined] - "chat_id": target_chat_id, - "text": msg, - "parse_mode": "HTML", - "disable_web_page_preview": True, - "reply_markup": keyboard, - }) + await tg.send_canonical_message( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + text=msg, + parse_mode="HTML", + reply_markup=keyboard, + ) except Exception as e: logger.warning("coverage_telegram_failed", error=str(e)) diff --git a/apps/api/src/jobs/hermes_rule_quality_job.py b/apps/api/src/jobs/hermes_rule_quality_job.py index cc20fa75a..769289104 100644 --- a/apps/api/src/jobs/hermes_rule_quality_job.py +++ b/apps/api/src/jobs/hermes_rule_quality_job.py @@ -312,14 +312,15 @@ async def _send_telegram_summary( """推 Telegram 摘要訊息給 SRE group,含 LLM 分析結果 + 互動按鈕 (P0 修).""" try: import html - from src.core.config import settings - from src.services.ai_advisory_helpers import build_ai_advisory_keyboard, is_snoozed - from src.services.telegram_gateway import get_telegram_gateway - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not target_chat_id: - logger.info("hermes_telegram_skip_no_chat_id") - return False + from src.services.ai_advisory_helpers import ( + build_ai_advisory_keyboard, + is_snoozed, + ) + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) # Snooze check: 以第一條 noisy rule_name 為 key primary_rule = noisy[0]["rule_name"] if noisy else "unknown" @@ -363,14 +364,15 @@ async def _send_telegram_summary( ) tg = get_telegram_gateway() - await tg._send_request("sendMessage", { # type: ignore[attr-defined] - "chat_id": target_chat_id, - "text": msg, - "parse_mode": "HTML", - "disable_web_page_preview": True, - "reply_markup": keyboard, - }) - return True + result = await tg.send_canonical_message( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + text=msg, + parse_mode="HTML", + reply_markup=keyboard, + ) + return _telegram_send_delivery_succeeded(result) except Exception as e: logger.warning("hermes_telegram_send_failed", error=str(e)) return False diff --git a/apps/api/src/services/ai_agent_report_truth_actionability_review.py b/apps/api/src/services/ai_agent_report_truth_actionability_review.py index 14050da87..ae42efa10 100644 --- a/apps/api/src/services/ai_agent_report_truth_actionability_review.py +++ b/apps/api/src/services/ai_agent_report_truth_actionability_review.py @@ -42,7 +42,7 @@ _BOT_TOKEN_URL_RE = re.compile( + r")\b", re.IGNORECASE, ) -_TELEGRAM_EGRESS_SCAN_SUFFIXES = {".py", ".sh", ".js", ".yml", ".yaml"} +_TELEGRAM_EGRESS_SCAN_SUFFIXES = {".py", ".ps1", ".sh", ".js", ".yml", ".yaml"} def load_latest_ai_agent_report_truth_actionability_review( @@ -188,6 +188,9 @@ def _telegram_egress_scan_roots(root: Path) -> list[Path]: roots = guard.get("guarded_roots") if isinstance(guard.get("guarded_roots"), list) else [] if not roots: roots = [".gitea/workflows", "scripts/ops", "scripts/ci", "apps/api/src"] + for required_root in ("agent99-control-plane.ps1", "scripts/reboot-recovery"): + if required_root not in roots: + roots.append(required_root) return [root / str(path) for path in roots] @@ -210,6 +213,10 @@ def _telegram_direct_surface_kind(relative_path: str) -> str: return "ci_script_direct_bot_api" if relative_path.startswith("apps/api/src/"): return "api_direct_bot_api" + if relative_path == "agent99-control-plane.ps1": + return "agent99_control_plane_direct_bot_api" + if relative_path.startswith("scripts/reboot-recovery/"): + return "reboot_recovery_direct_bot_api" return "unknown_direct_bot_api" @@ -222,6 +229,10 @@ def _telegram_direct_persistence_state(surface_kind: str) -> str: return "ci_script_log_only_no_awooop_db_receipt" if surface_kind == "api_direct_bot_api": return "api_direct_send_no_awooop_outbound_receipt_contract" + if surface_kind == "agent99_control_plane_direct_bot_api": + return "agent99_local_log_only_no_awooop_db_receipt" + if surface_kind == "reboot_recovery_direct_bot_api": + return "reboot_recovery_stdout_only_no_awooop_db_receipt" return "direct_send_no_awooop_outbound_receipt_contract" @@ -238,7 +249,8 @@ def _scan_current_direct_telegram_endpoints(root: Path) -> list[dict[str, Any]]: for scan_root in _telegram_egress_scan_roots(root): if not scan_root.exists(): continue - for path in sorted(scan_root.rglob("*")): + candidate_paths = [scan_root] if scan_root.is_file() else sorted(scan_root.rglob("*")) + for path in candidate_paths: if not path.is_file() or path.suffix not in _TELEGRAM_EGRESS_SCAN_SUFFIXES: continue relative_path = path.relative_to(root).as_posix() diff --git a/apps/api/src/services/ai_rate_limiter.py b/apps/api/src/services/ai_rate_limiter.py index 09b2eda28..87fee12a6 100644 --- a/apps/api/src/services/ai_rate_limiter.py +++ b/apps/api/src/services/ai_rate_limiter.py @@ -269,16 +269,23 @@ class AIRateLimiter: if await r.get(alert_sent_key): return - # 標記已發送 (24小時後可重新發送) - await r.set(alert_sent_key, "1", ex=86400) - try: - from src.core.config import settings - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not settings.OPENCLAW_TG_BOT_TOKEN or not target_chat_id: - logger.warning("telegram_not_configured_for_cost_alert") + gateway = get_telegram_gateway() + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="business_finops", + severity="P2", + ): + logger.warning( + "ai_cost_alert_no_send", + provider=provider, + reason="canonical_route_unavailable", + ) return message = ( @@ -291,22 +298,36 @@ class AIRateLimiter: f"redis-cli DEL ai_rate:total_cost:{provider}" ) - gateway = get_telegram_gateway() - await gateway._send_request( - "sendMessage", - { - "chat_id": target_chat_id, - "text": message, - "parse_mode": "HTML", - }, + result = await gateway.send_canonical_message( + product_id="awoooi", + signal_family="business_finops", + severity="P2", + text=message, + parse_mode="HTML", ) - logger.error( - "ai_cost_alert_sent", - provider=provider, - current_cost=f"${current_cost:.2f}", - limit=f"${limit:.2f}", - ) + if _telegram_send_delivery_succeeded(result): + await r.set(alert_sent_key, "1", ex=86400) + logger.warning( + "ai_cost_alert_sent", + provider=provider, + current_cost=f"${current_cost:.2f}", + limit=f"${limit:.2f}", + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) + else: + logger.warning( + "ai_cost_alert_no_send", + provider=provider, + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) except Exception as e: logger.error("ai_cost_alert_failed", error=str(e)) @@ -322,14 +343,23 @@ class AIRateLimiter: if await r.get(warning_key): return - await r.set(warning_key, "1", ex=3600) - try: - from src.core.config import settings - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not settings.OPENCLAW_TG_BOT_TOKEN or not target_chat_id: + gateway = get_telegram_gateway() + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="business_finops", + severity="P2", + ): + logger.warning( + "ai_cost_warning_no_send", + provider=provider, + reason="canonical_route_unavailable", + ) return limit = COST_LIMITS[provider]["total_cost_usd"] @@ -343,22 +373,35 @@ class AIRateLimiter: f"接近上限，請注意監控！" ) - gateway = get_telegram_gateway() - await gateway._send_request( - "sendMessage", - { - "chat_id": target_chat_id, - "text": message, - "parse_mode": "HTML", - }, + result = await gateway.send_canonical_message( + product_id="awoooi", + signal_family="business_finops", + severity="P2", + text=message, + parse_mode="HTML", ) - logger.warning( - "ai_cost_warning_sent", - provider=provider, - current_cost=f"${current_cost:.2f}", - threshold=f"${threshold:.2f}", - ) + if _telegram_send_delivery_succeeded(result): + await r.set(warning_key, "1", ex=3600) + logger.warning( + "ai_cost_warning_sent", + provider=provider, + current_cost=f"${current_cost:.2f}", + threshold=f"${threshold:.2f}", + ) + else: + logger.warning( + "ai_cost_warning_no_send", + provider=provider, + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) except Exception as e: logger.warning("ai_cost_warning_failed", error=str(e)) diff --git a/apps/api/src/services/ai_router.py b/apps/api/src/services/ai_router.py index 2e88fc307..5dffc0ce7 100644 --- a/apps/api/src/services/ai_router.py +++ b/apps/api/src/services/ai_router.py @@ -1354,7 +1354,13 @@ class AIRouterExecutor: "需要人工介入" ) _asyncio.create_task( - tg.send_alert_notification(formatted, parse_mode="HTML") + tg.send_alert_notification( + formatted, + parse_mode="HTML", + product_id="awoooi", + signal_family="shared_infrastructure", + severity="P1", + ) ) except Exception as _tg_e: logger.warning("diagnose_reject_telegram_failed", error=str(_tg_e)) diff --git a/apps/api/src/services/approval_execution.py b/apps/api/src/services/approval_execution.py index c609957ef..8cf1058e4 100644 --- a/apps/api/src/services/approval_execution.py +++ b/apps/api/src/services/approval_execution.py @@ -1127,17 +1127,49 @@ class ApprovalExecutionService: approval_id=str(approval.id), ) - from src.core.config import get_settings - from src.services.telegram_gateway import get_telegram_gateway - settings = get_settings() + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() - target_chat_id = settings.SRE_GROUP_CHAT_ID + target_chat_id = gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) if not target_chat_id: logger.warning( "push_execution_result_no_target_chat", incident_id=approval.incident_id, approval_id=str(approval.id), ) + try: + from src.repositories.alert_operation_log_repository import ( + get_alert_operation_log_repository, + ) + + await get_alert_operation_log_repository().append( + "NOTIFICATION_CLASSIFIED", + incident_id=approval.incident_id, + approval_id=str(approval.id), + actor="approval_execution", + action_detail="telegram_execution_result_no_send", + success=False, + error_message="canonical_route_unavailable", + context={ + "delivery_status": "blocked_no_route", + "execution_success": success, + "execution_kind": execution_kind, + "repair_executed": repair_executed, + "repair_attempted": repair_attempted, + }, + ) + except Exception as _log_e: + logger.warning( + "alert_op_telegram_result_no_send_write_failed", + approval_id=str(approval.id), + error=str(_log_e), + ) return # 2026-04-19 ogt + Claude Opus 4.7 修 AP-2: 除了 reply 外, @@ -1157,6 +1189,7 @@ class ApprovalExecutionService: km_info = "" try: from sqlalchemy import text as _sql + from src.db.base import get_db_context async with get_db_context() as _db: _km_row = await _db.execute( @@ -1187,20 +1220,20 @@ class ApprovalExecutionService: truth_chain_row = incident_truth_chain_button_row( approval.incident_id or "" ) - payload: dict[str, Any] = { - "chat_id": target_chat_id, - "text": text, - "parse_mode": "HTML", - "disable_web_page_preview": True, - } - if truth_chain_row: - payload["reply_markup"] = { - "inline_keyboard": [truth_chain_row], - } - if orig_msg_id is not None: - payload["reply_to_message_id"] = orig_msg_id - - await gateway._send_request("sendMessage", payload) + delivery_result = await gateway.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + parse_mode="HTML", + reply_markup=( + {"inline_keyboard": [truth_chain_row]} + if truth_chain_row + else None + ), + reply_to_message_id=orig_msg_id, + ) + delivery_succeeded = _telegram_send_delivery_succeeded(delivery_result) context_execution_kind = execution_kind or ( "no_action" if no_action else "execution" ) @@ -1209,8 +1242,22 @@ class ApprovalExecutionService: if repair_executed is not None else bool(success and not no_action) ) - logger.info( - "push_execution_result_sent", + delivery_status = str( + ( + delivery_result.get("_awooop_delivery_status") + if isinstance(delivery_result, dict) + else None + ) + or "missing_provider_ack" + ) + log_event = ( + "push_execution_result_sent" + if delivery_succeeded + else "push_execution_result_no_send" + ) + log_method = logger.info if delivery_succeeded else logger.warning + log_method( + log_event, incident_id=approval.incident_id, approval_id=str(approval.id), success=success, @@ -1220,6 +1267,7 @@ class ApprovalExecutionService: repair_executed=context_repair_executed, operator_outcome_state=outcome.get("state"), needs_human=outcome.get("needs_human"), + delivery_status=delivery_status, ) try: from src.repositories.alert_operation_log_repository import ( @@ -1227,13 +1275,25 @@ class ApprovalExecutionService: ) await get_alert_operation_log_repository().append( - "TELEGRAM_RESULT_SENT", + ( + "TELEGRAM_RESULT_SENT" + if delivery_succeeded + else "NOTIFICATION_CLASSIFIED" + ), incident_id=approval.incident_id, approval_id=str(approval.id), actor="approval_execution", - action_detail="telegram_execution_result_sent", - success=success, - error_message=error, + action_detail=( + "telegram_execution_result_sent" + if delivery_succeeded + else "telegram_execution_result_no_send" + ), + success=(success if delivery_succeeded else False), + error_message=( + error + if delivery_succeeded + else delivery_status or "missing_provider_ack" + ), context={ "reply_to_message_id": orig_msg_id, "execution_kind": context_execution_kind, @@ -1241,6 +1301,9 @@ class ApprovalExecutionService: "repair_attempted": repair_attempted, "operator_outcome": outcome, "delivery": "reply" if orig_msg_id is not None else "standalone", + "delivery_status": delivery_status, + "provider_delivery_acknowledged": delivery_succeeded, + "execution_success": success, }, ) except Exception as _log_e: diff --git a/apps/api/src/services/channel_hub.py b/apps/api/src/services/channel_hub.py index 83fc28a83..43db5b280 100644 --- a/apps/api/src/services/channel_hub.py +++ b/apps/api/src/services/channel_hub.py @@ -1014,6 +1014,7 @@ async def schedule_interim_feedback( run_id: UUID, channel_type: str, channel_chat_id: str, + inbound_message_id: int | None = None, conversation_event_id: UUID | None = None, is_shadow: bool = True, wait_seconds: int = _INTERIM_WAIT_SECONDS, @@ -1030,6 +1031,7 @@ async def schedule_interim_feedback( run_id=run_id, channel_type=channel_type, channel_chat_id=channel_chat_id, + inbound_message_id=inbound_message_id, conversation_event_id=conversation_event_id, is_shadow=is_shadow, wait_seconds=wait_seconds, @@ -1044,6 +1046,7 @@ async def _interim_feedback_task( run_id: UUID, channel_type: str, channel_chat_id: str, + inbound_message_id: int | None, conversation_event_id: UUID | None, is_shadow: bool, wait_seconds: int, @@ -1095,14 +1098,26 @@ async def _interim_feedback_task( if not (is_shadow or run_is_shadow): # Non-shadow：實際發 Telegram 訊息 - await _send_telegram_interim( + delivered = await _send_telegram_interim( channel_chat_id=channel_chat_id, content=interim_content, run_id=run_id, + inbound_message_id=inbound_message_id, ) + if not delivered: + logger.warning( + "interim_feedback_blocked_no_send", + project_id=project_id, + run_id=str(run_id), + ) + return logger.info( - "interim_feedback_sent", + ( + "interim_feedback_shadow_recorded" + if is_shadow or run_is_shadow + else "interim_feedback_sent" + ), project_id=project_id, run_id=str(run_id), is_shadow=is_shadow or run_is_shadow, @@ -1121,23 +1136,45 @@ async def _send_telegram_interim( channel_chat_id: str, content: str, run_id: UUID, -) -> None: + inbound_message_id: int | None = None, +) -> bool: """透過 TelegramGateway 發送 interim 訊息（non-shadow 專用）。""" try: - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) - await get_telegram_gateway().send_text( + if inbound_message_id is None or inbound_message_id <= 0: + logger.warning( + "interim_telegram_missing_verified_parent_no_send", + run_id=str(run_id), + ) + return False + + result = await get_telegram_gateway().send_text( content, chat_id=channel_chat_id, parse_mode="HTML", disable_web_page_preview=True, + interaction_kind="operator_command_reply", + inbound_chat_id=channel_chat_id, + inbound_message_id=inbound_message_id, ) + if not _telegram_send_delivery_succeeded(result): + logger.warning( + "interim_telegram_delivery_not_proven_no_send", + run_id=str(run_id), + ) + return False + return True except Exception as exc: logger.warning( "interim_telegram_send_failed", run_id=str(run_id), error=str(exc), ) + return False # ───────────────────────────────────────────────────────────────────────────── @@ -1189,11 +1226,16 @@ async def handle_telegram_inbound( # Step 3: Progressive Feedback（30s 計時器） if not is_duplicate: + try: + inbound_message_id = int(message_id) + except (TypeError, ValueError): + inbound_message_id = None await schedule_interim_feedback( project_id=project_id, run_id=run_id, channel_type="telegram", channel_chat_id=chat_id, + inbound_message_id=inbound_message_id, conversation_event_id=event_id, is_shadow=is_shadow, ) diff --git a/apps/api/src/services/converged_alert_recurrence_notifier.py b/apps/api/src/services/converged_alert_recurrence_notifier.py index 054f0330c..41533f12c 100644 --- a/apps/api/src/services/converged_alert_recurrence_notifier.py +++ b/apps/api/src/services/converged_alert_recurrence_notifier.py @@ -5,7 +5,10 @@ import html from src.core.logging import get_logger from src.core.redis_client import get_redis -from src.services.telegram_gateway import get_telegram_gateway +from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) logger = get_logger("awoooi.converged_alert_recurrence") @@ -67,6 +70,22 @@ async def should_notify_converged_alert_recurrence( return is_converged_alert_recurrence_milestone(hit_count) +async def _release_converged_alert_recurrence_reservation( + fingerprint: str, +) -> None: + """Release a pre-send reservation when provider delivery is not proven.""" + + key = converged_alert_recurrence_key(fingerprint) + try: + await get_redis().delete(key) + except Exception as exc: + logger.warning( + "converged_alert_recurrence_reservation_release_failed", + fingerprint_hash=key.rsplit(":", 1)[-1], + error=str(exc), + ) + + def format_converged_alert_recurrence_message( *, source: str, @@ -166,9 +185,23 @@ async def notify_converged_alert_recurrence( failures: list[str] = [] try: - await gateway.send_alert_notification(text) - sent_count += 1 + result = await gateway.send_alert_notification( + text, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + if _telegram_send_delivery_succeeded(result): + sent_count += 1 + else: + await _release_converged_alert_recurrence_reservation(fingerprint) + result_payload = result if isinstance(result, dict) else {} + receipt = result_payload.get("_awoooi_canonical_route_receipt") or {} + failures.append( + f"route:{receipt.get('classifier') or 'blocked_no_egress'}" + ) except Exception as exc: + await _release_converged_alert_recurrence_reservation(fingerprint) failures.append(f"primary:{type(exc).__name__}") logger.warning( "converged_alert_recurrence_primary_failed", diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index e59d3a3f9..afbe5964f 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -195,19 +195,57 @@ async def _send_log_summary(incident: "Incident") -> None: if incident.signals: namespace = incident.signals[0].labels.get("namespace", "awoooi-prod") + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) + + tg = get_telegram_gateway() + if not tg.canonical_destination_chat_id( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ): + logger.warning( + "log_summary_no_send", + target=target, + incident_id=incident.incident_id, + reason="canonical_route_unavailable", + ) + return + from src.services.log_summary_service import get_log_summary_service svc = get_log_summary_service() summary = await svc.summarize_with_soft_timeout(pod_name=target, namespace=namespace) if not summary: return - from src.services.telegram_gateway import get_telegram_gateway - tg = get_telegram_gateway() - await tg.send_as_nemotron( - f"📋 Log 異常摘要 — {target}\n{summary}" + result = await tg.send_as_nemotron( + f"📋 Log 異常摘要 — {target}\n{summary}", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) - import structlog as _sl - _sl.get_logger(__name__).info("log_summary_sent", target=target, incident_id=incident.incident_id) + if _telegram_send_delivery_succeeded(result): + logger.info( + "log_summary_sent", + target=target, + incident_id=incident.incident_id, + ) + else: + logger.warning( + "log_summary_no_send", + target=target, + incident_id=incident.incident_id, + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) except Exception as e: import structlog as _sl _sl.get_logger(__name__).warning("log_summary_failed", error=str(e)) @@ -220,7 +258,7 @@ async def _send_log_summary(incident: "Incident") -> None: async def _push_decision_to_telegram( incident: Incident, proposal_data: dict[str, Any], -) -> None: +) -> bool: """ 決策就緒時推送到 Telegram @@ -231,11 +269,12 @@ async def _push_decision_to_telegram( # 延遲導入避免循環依賴 from src.core.redis_client import get_redis from src.services.telegram_gateway import ( + NotificationType, + _telegram_send_delivery_succeeded, classify_notification, get_telegram_gateway, - NotificationType, - _smart_truncate as _smt, ) + from src.services.telegram_gateway import _smart_truncate as _smt # 🔴 去重檢查：同一 fingerprint (alertname+target) 24h 內只發一次 decision card # 2026-05-02 Claude Opus 4.7 + 統帥 ogt：原本 dedup_key 用 incident.incident_id， @@ -256,7 +295,7 @@ async def _push_decision_to_telegram( incident_id=incident.incident_id, fingerprint=f"{_alertname_fp}:{_target_fp}", ) - return + return False # 2026-04-09 Claude Code: resolved Incident 不重送 Telegram # 場景: dedup TTL 過期後，已 resolve 的 Incident 仍被重新推送 @@ -266,7 +305,7 @@ async def _push_decision_to_telegram( reason="Incident already resolved", incident_id=incident.incident_id, ) - return + return False # 🔴 靜默檢查：此資源是否被靜默 (2026-03-27 P1 優化) target = incident.affected_services[0] if incident.affected_services else "unknown" @@ -278,7 +317,7 @@ async def _push_decision_to_telegram( incident_id=incident.incident_id, resource=target, ) - return + return False # 檢查是否有設定 Bot Token if not settings.OPENCLAW_TG_BOT_TOKEN: @@ -287,7 +326,7 @@ async def _push_decision_to_telegram( reason="Bot token not configured", incident_id=incident.incident_id, ) - return + return False gateway = get_telegram_gateway() @@ -411,7 +450,7 @@ async def _push_decision_to_telegram( action_text=_action_text, reason="Agent Debate 無有效輸出 (description=待分析, action 屬於失敗狀態集合)", ) - return + return False # 建立 approval_id (使用 incident_id 作為追蹤) # 2026-03-27 ogt: 修復 INC-INC-INC- 重複前綴 bug @@ -581,12 +620,37 @@ async def _push_decision_to_telegram( automation_state=proposal_data.get("automation_state", ""), ) + if not _telegram_send_delivery_succeeded(tg_result): + logger.warning( + "telegram_decision_no_send", + incident_id=incident.incident_id, + source=source, + risk_level=risk_level, + delivery_status=( + tg_result.get("_awooop_delivery_status") + if isinstance(tg_result, dict) + else "missing_provider_ack" + ), + ) + return False + # 2026-04-09 Claude Sonnet 4.6: 存 message_id → 後續狀態更新在原訊息延續 # 同時寫 Redis (快速查詢) 和 DB (持久化，不受 TTL 限制) tg_message_id = tg_result.get("result", {}).get("message_id") if isinstance(tg_result, dict) else None tg_chat_id = tg_result.get("result", {}).get("chat", {}).get("id") if isinstance(tg_result, dict) else None if tg_message_id: - await redis.setex(f"tg_msg:{incident.incident_id}", 86400, str(tg_message_id)) + try: + await redis.setex( + f"tg_msg:{incident.incident_id}", + 86400, + str(tg_message_id), + ) + except Exception as _redis_message_error: + logger.warning( + "telegram_message_id_redis_save_failed", + incident_id=incident.incident_id, + error=str(_redis_message_error), + ) # 持久化到 DB try: from src.services.approval_db import get_approval_service as _get_approval_svc @@ -599,6 +663,21 @@ async def _push_decision_to_telegram( except Exception as _e: logger.warning("telegram_message_id_db_save_failed", incident_id=incident.incident_id, error=str(_e)) + # Provider ack is the delivery truth. Persist dedup only after that + # receipt, while keeping a dedup write failure separate from delivery. + # 2026-05-02 Claude Opus 4.7 + 統帥 ogt：TTL 從 600s 拉到 86400s； + # incident_analysis_sweeper 每 1h 與 Alertmanager 每 4h 都可能重觸， + # 24h TTL 用來避免同症狀重複洗版。 + try: + await redis.setex(dedup_key, 86400, "1") + except Exception as _dedup_error: + logger.warning( + "telegram_decision_dedup_write_failed", + incident_id=incident.incident_id, + dedup_key=dedup_key, + error=str(_dedup_error), + ) + # Phase 31 (ADR-067 2026-04-10): Log 異常摘要 — NemoTron deepseek-r1:14b # 非同步執行，不阻塞主流程 _fire_and_forget(_send_log_summary(incident)) @@ -610,22 +689,20 @@ async def _push_decision_to_telegram( _fire_and_forget( gateway.send_as_nemotron( f"🤔 NemoClaw 第二意見 (信心={confidence:.2f})\n" - f"{_advisory_note}" + f"{_advisory_note}", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) ) - # 🔴 發送成功後設置去重 key (TTL 24 小時) - # 2026-05-02 Claude Opus 4.7 + 統帥 ogt：TTL 從 600s 拉到 86400s。 - # 原因：incident_analysis_sweeper 每 1h 重觸 decision、alertmanager 預設 4h repeat， - # 600s TTL 早就過期 → 重發；24h TTL 蓋住兩個週期，徹底治住「同症狀 24h 推 15 次」。 - await redis.setex(dedup_key, 86400, "1") - logger.info( "telegram_decision_pushed", incident_id=incident.incident_id, source=source, risk_level=risk_level, ) + return True except Exception as e: # Telegram 失敗不影響主流程 @@ -634,6 +711,7 @@ async def _push_decision_to_telegram( incident_id=incident.incident_id, error=str(e), ) + return False async def _nemoclaw_second_opinion(incident: "Incident", primary_result: dict) -> str | None: @@ -831,7 +909,10 @@ async def _generate_playbook_draft_if_new(incident: "Incident") -> None: f"📚 KM 新條目 — {entry.entry_id}\n" f"🏷️ 分類：auto_generated 狀態：DRAFT\n" f"📋 告警：{alertname} 事件：{incident.incident_id}\n" - f"標題：{str(getattr(entry, 'title', alertname))[:60]}" + f"標題：{str(getattr(entry, 'title', alertname))[:60]}", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) except Exception as _tg_err: _sl.get_logger(__name__).debug("km_tg_notify_failed", error=str(_tg_err)) @@ -1110,7 +1191,10 @@ async def _push_auto_repair_result( 2026-04-11 Claude Sonnet 4.6: +ADR-071-I 指標快照 """ try: - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() target = incident.affected_services[0] if incident.affected_services else "unknown" @@ -1217,9 +1301,57 @@ async def _push_auto_repair_result( f"├─ 對象: {target[:50]}\n" f"└─ {status_line}" ) - await gateway.send_alert_notification(fallback_text) + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ): + logger.warning( + "auto_repair_result_no_send", + incident_id=inc_id, + success=success, + reason="canonical_route_unavailable", + ) + return - logger.info("auto_repair_result_sent", incident_id=inc_id, success=success, appended=appended) + fallback_result = await gateway.send_alert_notification( + fallback_text, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + if not _telegram_send_delivery_succeeded(fallback_result): + logger.warning( + "auto_repair_result_no_send", + incident_id=inc_id, + success=success, + delivery_status=( + fallback_result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(fallback_result, dict) + else "missing_provider_ack" + ), + ) + return + + logger.info( + "auto_repair_result_sent", + incident_id=inc_id, + success=success, + appended=False, + ) + else: + # append_incident_update currently exposes only a boolean request result, + # not the provider receipt. Do not claim a Telegram send without the + # shared affirmative delivery verifier. + logger.info( + "auto_repair_result_update_requested", + incident_id=inc_id, + success=success, + appended=True, + ) except Exception as e: logger.warning("auto_repair_result_push_failed", incident_id=incident.incident_id, error=str(e)) @@ -3506,32 +3638,45 @@ class DecisionManager: """ import asyncio as _asyncio import json as _json + from src.core.redis_client import get_redis from src.db.base import get_db_context from src.repositories.incident_repository import IncidentDBRepository redis = get_redis() - resent = 0 # GAP-A4 後續修復：限制並發 5，避免壓爆 Ollama _sem = _asyncio.Semaphore(5) - async def _bounded_push(incident_obj, proposal_data_obj, _id, _token): + async def _bounded_push(incident_obj, proposal_data_obj, _id, _token) -> bool: async with _sem: try: - await _push_decision_to_telegram(incident_obj, proposal_data_obj) - logger.info( - "stale_ready_token_resent", - incident_id=_id, - token=_token, + delivered = await _push_decision_to_telegram( + incident_obj, + proposal_data_obj, ) + if delivered: + logger.info( + "stale_ready_token_resent", + incident_id=_id, + token=_token, + ) + else: + logger.warning( + "stale_ready_token_no_send", + incident_id=_id, + token=_token, + ) + return delivered except Exception as _e: logger.warning( "stale_ready_token_resend_failed", incident_id=_id, error=str(_e), ) - # 每次完成後喘 200ms，給 Ollama embedding 恢復空間 - await _asyncio.sleep(0.2) + return False + finally: + # 每次完成後喘 200ms，給 Ollama embedding 恢復空間 + await _asyncio.sleep(0.2) try: # 掃描所有 decision:* key @@ -3575,7 +3720,10 @@ class DecisionManager: _STALE_DAYS = 3 _created_at = getattr(incident, "created_at", None) if _created_at: - from datetime import datetime as _dt, timedelta as _td, timezone as _tz + from datetime import datetime as _dt + from datetime import timedelta as _td + from datetime import timezone as _tz + _now = _dt.now(_tz.utc) _cutoff = _now - _td(days=_STALE_DAYS) # 確保 _created_at 有時區 @@ -3599,20 +3747,26 @@ class DecisionManager: _bounded_push(incident, proposal_data, incident_id, data.get("token", "")) ) tasks.append(_task) - resent += 1 except Exception as _te: logger.debug("stale_ready_token_scan_error", error=str(_te)) if cursor == 0: break - # 不等所有 task 完成（fire-and-forget 語義保留），但 await 一下讓並發限制生效 + resent = 0 if tasks: - logger.info("stale_ready_tokens_throttled_dispatch", - total=len(tasks), max_concurrent=5) + logger.info( + "stale_ready_tokens_throttled_dispatch", + total=len(tasks), + max_concurrent=5, + status="awaiting_delivery_receipts", + ) + delivery_results = await _asyncio.gather(*tasks) + resent = sum(result is True for result in delivery_results) except Exception as e: logger.warning("resend_stale_ready_tokens_failed", error=str(e)) + resent = 0 logger.info("stale_ready_tokens_scan_done", resent=resent) return resent diff --git a/apps/api/src/services/drift_adopt_service.py b/apps/api/src/services/drift_adopt_service.py index 6229e3845..3dbd61429 100644 --- a/apps/api/src/services/drift_adopt_service.py +++ b/apps/api/src/services/drift_adopt_service.py @@ -477,7 +477,10 @@ class DriftAdoptService: f"Namespace: {report.namespace}\n" f"漂移: {report.summary}\n\n" f"PR: {pr_url}\n\n" - f"請 SRE review 後 merge。" + f"請 SRE review 後 merge。", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", ) except Exception as e: logger.warning("drift_adopt_telegram_failed", error=str(e)) diff --git a/apps/api/src/services/drift_remediator.py b/apps/api/src/services/drift_remediator.py index 514fe5113..19bb51f46 100644 --- a/apps/api/src/services/drift_remediator.py +++ b/apps/api/src/services/drift_remediator.py @@ -224,7 +224,12 @@ class DriftRemediator: try: from src.services.telegram_gateway import get_telegram_gateway tg = get_telegram_gateway() - await tg.send_text(message) + await tg.send_text( + message, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) except Exception as e: logger.warning("drift_remediator_telegram_failed", error=str(e)) diff --git a/apps/api/src/services/emergency_escalation_service.py b/apps/api/src/services/emergency_escalation_service.py index 4a8a15b6c..cc809bda9 100644 --- a/apps/api/src/services/emergency_escalation_service.py +++ b/apps/api/src/services/emergency_escalation_service.py @@ -10,7 +10,6 @@ from typing import Any import structlog -from src.core.config import settings from src.core.redis_client import get_redis logger = structlog.get_logger(__name__) @@ -78,7 +77,6 @@ async def escalate_auto_repair_unavailable( attempted_actions=attempted_actions, failure_reason=failure_reason, current_impact=f"target={target_resource or 'unknown'} namespace={namespace or 'unknown'}", - group_chat_id=settings.SRE_GROUP_CHAT_ID or None, ) await get_alert_operation_log_repository().append( @@ -177,7 +175,6 @@ async def escalate_drift_auto_adopt_blocked( f"intent={intent} confidence={confidence:.0%} risk={risk} " f"fingerprint={fingerprint}" ), - group_chat_id=settings.SRE_GROUP_CHAT_ID or None, ) await get_alert_operation_log_repository().append( "APPROVAL_ESCALATED", diff --git a/apps/api/src/services/failover_alerter.py b/apps/api/src/services/failover_alerter.py index 3c9456424..ad4f825d5 100644 --- a/apps/api/src/services/failover_alerter.py +++ b/apps/api/src/services/failover_alerter.py @@ -68,8 +68,15 @@ class FailoverAlerter: f"Fallback 鏈：{_escape_md(fallback_chain_str)}\n\n" f"自動恢復服務持續監控，3 次 HEALTHY 後自動切回" ) - await self._send(msg) - logger.info("failover_alert_sent", to_provider=to_provider) + delivered = await self._deliver_reserved( + msg, + dedup_key, + ttl=DEDUP_TTL_SEC, + ) + if delivered: + logger.info("failover_alert_sent", to_provider=to_provider) + else: + logger.warning("failover_alert_no_send", to_provider=to_provider) async def alert_recovery(self, event: dict[str, Any]) -> None: """Ollama 自動恢復告警 — 1h dedup per host @@ -99,8 +106,15 @@ class FailoverAlerter: f"切換路徑：{_escape_md(str(from_provider))} → {_escape_md(str(to_provider))}\n\n" f"自動化飛輪已恢復至高效能推理模式" ) - await self._send(msg) - logger.info("recovery_alert_sent", from_provider=from_provider) + delivered = await self._deliver_reserved( + msg, + dedup_key, + ttl=RECOVERY_DEDUP_TTL_SEC, + ) + if delivered: + logger.info("recovery_alert_sent", from_provider=from_provider) + else: + logger.warning("recovery_alert_no_send", from_provider=from_provider) async def alert_governance(self, event_type: str, payload: dict[str, Any]) -> None: """AI 治理告警（dedup 1h） @@ -135,8 +149,11 @@ class FailoverAlerter: return msg = format_governance_alert_card(event_type, payload) - await self._send(msg) - logger.info("governance_alert_sent", event_type=event_type) + delivered = await self._deliver_reserved(msg, dedup_key, ttl=3600) + if delivered: + logger.info("governance_alert_sent", event_type=event_type) + else: + logger.warning("governance_alert_no_send", event_type=event_type) async def alert_gemini_quota_exceeded(self, event: dict[str, Any]) -> None: """Gemini 每日上限觸發，降級到 188 CPU 備援 — 24h dedup（每日重置）""" @@ -161,8 +178,15 @@ class FailoverAlerter: f"進入容災模式至明日 0:00\n" f"建議檢查是否有異常流量，評估是否升級 Gemini 配額" ) - await self._send(msg) - logger.info("quota_alert_sent", quota=quota, current_count=current_count) + delivered = await self._deliver_reserved( + msg, + dedup_key, + ttl=QUOTA_DEDUP_TTL_SEC, + ) + if delivered: + logger.info("quota_alert_sent", quota=quota, current_count=current_count) + else: + logger.warning("quota_alert_no_send", quota=quota, current_count=current_count) async def alert_provider_version_changed(self, changed_providers: list[str], probed: int) -> None: """AI Provider 版本變更告警 — dedup 1h/provider @@ -171,19 +195,20 @@ class FailoverAlerter: 每個 provider 獨立 dedup，避免同一版本重複告警。 """ now_str = datetime.now(TAIPEI_TZ).strftime("%Y-%m-%d %H:%M") - sent: list[str] = [] + reserved: list[tuple[str, str]] = [] for provider in changed_providers: dedup_key = f"alert:provider_version_changed:{provider}" if not await self._check_dedup(dedup_key, ttl=3600): logger.debug("provider_version_alert_dedup_skipped", provider=provider) continue - sent.append(provider) + reserved.append((provider, dedup_key)) - if not sent: + if not reserved: return - providers_md = "\n".join(f"• {_escape_md(p)}" for p in sent) + providers = [provider for provider, _dedup_key in reserved] + providers_md = "\n".join(f"• {_escape_md(p)}" for p in providers) msg = ( f"*AI Provider 版本變更偵測*\n\n" f"時間：{_escape_md(now_str)}\n" @@ -191,8 +216,20 @@ class FailoverAlerter: f"版本已變更：\n{providers_md}\n\n" f"系統已自動記錄版本歷史，請確認是否需要重新驗證推理品質" ) - await self._send(msg) - logger.info("provider_version_alert_sent", sent=sent) + delivered = False + try: + delivered = await self._send(msg) + finally: + for _provider, dedup_key in reserved: + await self._finalize_dedup( + dedup_key, + ttl=3600, + delivered=delivered, + ) + if delivered: + logger.info("provider_version_alert_sent", sent=providers) + else: + logger.warning("provider_version_alert_no_send", providers=providers) # ------------------------------------------------------------------------- # Dedup（Redis SET NX EX） @@ -200,8 +237,12 @@ class FailoverAlerter: async def _check_dedup(self, key: str, ttl: int) -> bool: """ - Redis SET NX EX 防止重複告警。 - True = 第一次（應送出），False = 已送過（跳過）。 + Redis SET NX EX 建立 delivery reservation。 + True = 可嘗試送出，False = 已送過或另一個 worker 正在送。 + + reservation 的 value 是 ``pending``，不是 sent receipt。外層只有收到 + affirmative provider ack 後才透過 ``_finalize_dedup`` 改成 ``sent``； + no-send / exception 會刪除 reservation，讓下一輪可以安全重試。 2026-04-25 P1.5 by Claude Engineer-D — Telegram dedup 鐵律 10min/24h TTL 2026-04-27 Wave8-X2 by Claude — dedup fail-open 修復 @@ -212,7 +253,12 @@ class FailoverAlerter: # 優先嘗試 Redis if self._redis is not None: try: - ok = await self._redis.set(f"{key}:dedup", "1", ex=ttl, nx=True) + ok = await self._redis.set( + f"{key}:dedup", + "pending", + ex=ttl, + nx=True, + ) return bool(ok) except Exception as e: logger.warning("dedup_redis_failed_using_memory", error=str(e)) @@ -233,11 +279,63 @@ class FailoverAlerter: self._memory_dedup[key] = now return True + async def _finalize_dedup( + self, + key: str, + *, + ttl: int, + delivered: bool, + ) -> None: + """Commit sent dedup only after provider ack, otherwise release it. + + 2026-07-14 Codex v1.1 (Asia/Taipei): route acceptance and ``ok`` are + not delivery truth. Pending reservations prevent concurrent sends but + must never survive a structured no-send receipt as a terminal success. + """ + redis_finalized = False + if self._redis is not None: + try: + if delivered: + await self._redis.set(f"{key}:dedup", "sent", ex=ttl) + else: + await self._redis.delete(f"{key}:dedup") + redis_finalized = True + except Exception as e: + logger.warning( + "dedup_finalize_redis_failed_using_memory", + delivered=delivered, + error=str(e), + ) + + if delivered: + if not redis_finalized: + import time + + self._memory_dedup[key] = time.time() + else: + self._memory_dedup.pop(key, None) + + async def _deliver_reserved( + self, + message: str, + key: str, + *, + ttl: int, + ) -> bool: + """Deliver one reserved alert and always finalize or release its state.""" + + delivered = False + try: + delivered = await self._send(message) + return delivered + finally: + await self._finalize_dedup(key, ttl=ttl, delivered=delivered) + # ------------------------------------------------------------------------- # 發送（透過 TelegramGateway singleton） # ------------------------------------------------------------------------- - async def _send(self, message: str) -> None: + async def _send(self, message: str) -> bool: """發送至 Telegram SRE_GROUP_CHAT_ID 使用現有 TelegramGateway singleton（不另建 HTTP client）， @@ -248,18 +346,34 @@ class FailoverAlerter: 2026-04-25 P1.5 by Claude Engineer-D — 告警失敗不能阻斷主流程 """ try: - from src.core.config import get_settings - from src.services.telegram_gateway import get_telegram_gateway - - settings = get_settings() - chat_id = getattr(settings, "SRE_GROUP_CHAT_ID", None) - if not chat_id: - logger.warning("telegram_chat_id_missing_failover_alert") - return + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() - await gateway.send_alert_notification(text=message, parse_mode="MarkdownV2") - logger.info("telegram_failover_alert_sent", message_len=len(message)) + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="shared_infrastructure", + severity="P1", + ): + logger.warning("telegram_canonical_route_unavailable_failover_alert") + return False + delivery = await gateway.send_alert_notification( + text=message, + parse_mode="MarkdownV2", + product_id="awoooi", + signal_family="shared_infrastructure", + severity="P1", + ) + if _telegram_send_delivery_succeeded(delivery): + return True + else: + logger.warning( + "telegram_failover_alert_blocked_no_send", + message_len=len(message), + ) + return False except Exception as e: # 不 raise — 告警失敗不該阻斷主流程（鐵律） # 2026-05-06 Codex: Telegram/httpx exception 字串可能包含 bot token URL， @@ -269,6 +383,7 @@ class FailoverAlerter: error=_sanitize_telegram_error(str(e)), error_type=type(e).__name__, ) + return False # ------------------------------------------------------------------------- diff --git a/apps/api/src/services/failure_watcher.py b/apps/api/src/services/failure_watcher.py index e6b537e67..bd137019d 100644 --- a/apps/api/src/services/failure_watcher.py +++ b/apps/api/src/services/failure_watcher.py @@ -732,7 +732,10 @@ class FailureWatcherService(IFailureWatcher): ) # 推送到 Telegram - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) tg = get_telegram_gateway() message = ( @@ -744,13 +747,24 @@ class FailureWatcherService(IFailureWatcher): f"└ 💡 下一步: {analysis.get('suggested_repair', 'AI 深度診斷')}\n\n" f"狀態：ai_retry_or_break_glass_recorded" ) - await tg.send_alert_notification(message) - - logger.info( - "ai_controlled_repair_followup_sent", - audit_log_id=audit_log_id, - risk_level=analysis.get("risk_level"), + delivery = await tg.send_alert_notification( + message, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", ) + if _telegram_send_delivery_succeeded(delivery): + logger.info( + "ai_controlled_repair_followup_sent", + audit_log_id=audit_log_id, + risk_level=analysis.get("risk_level"), + ) + else: + logger.warning( + "ai_controlled_repair_followup_blocked_no_send", + audit_log_id=audit_log_id, + risk_level=analysis.get("risk_level"), + ) except Exception as e: logger.warning( @@ -776,7 +790,12 @@ class FailureWatcherService(IFailureWatcher): f"├ 📋 AuditLog: {audit_log_id[:8]}...\n" f"└ 📝 結果: {repair_result}" ) - await tg.send_alert_notification(message) + await tg.send_alert_notification( + message, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) except Exception as e: logger.warning( diff --git a/apps/api/src/services/gitea_webhook_service.py b/apps/api/src/services/gitea_webhook_service.py index a9a697e6f..d092b03d2 100644 --- a/apps/api/src/services/gitea_webhook_service.py +++ b/apps/api/src/services/gitea_webhook_service.py @@ -406,9 +406,20 @@ class GiteaWebhookService: message = "\n".join(message_lines) - await telegram.send_alert_notification(message) + result = await telegram.send_alert_notification( + message, + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) - logger.info("gitea_telegram_sent", review_id=review_id, repo=repo, event_type=event_type) + logger.info( + "gitea_telegram_routed", + review_id=review_id, + repo=repo, + event_type=event_type, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) except Exception as e: logger.exception("gitea_telegram_failed", error=str(e)) diff --git a/apps/api/src/services/image_analysis_service.py b/apps/api/src/services/image_analysis_service.py index 7e30431fd..442799a4d 100644 --- a/apps/api/src/services/image_analysis_service.py +++ b/apps/api/src/services/image_analysis_service.py @@ -171,6 +171,7 @@ class ImageAnalysisService: self, chat_id: str, file_id: str, + original_message_id: int | None = None, question: str = "請用繁體中文描述這張圖片，如果是截圖或介面請分析其內容和問題", ) -> None: """ @@ -192,7 +193,11 @@ class ImageAnalysisService: from src.services.telegram_gateway import get_telegram_gateway tg = get_telegram_gateway() await tg.initialize() - await tg.send_as_openclaw(f"🖼️ 圖片分析\n{result}") + await tg.send_as_openclaw( + f"🖼️ 圖片分析\n{result}", + reply_to_message_id=original_message_id, + inbound_chat_id=chat_id, + ) except Exception as e: logger.warning("download_and_analyze_failed", error=str(e)) diff --git a/apps/api/src/services/k3s_monitor_service.py b/apps/api/src/services/k3s_monitor_service.py index 3f8d592c2..21ace7cf3 100644 --- a/apps/api/src/services/k3s_monitor_service.py +++ b/apps/api/src/services/k3s_monitor_service.py @@ -21,13 +21,17 @@ K3s Monitor Service - Phase 21.2 定期報告 import asyncio from datetime import datetime from typing import Any, Protocol, runtime_checkable +from zoneinfo import ZoneInfo import httpx import structlog -from zoneinfo import ZoneInfo from src.core.config import settings -from src.services.telegram_gateway import K3sStatusMessage, get_telegram_gateway +from src.services.telegram_gateway import ( + K3sStatusMessage, + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) logger = structlog.get_logger(__name__) @@ -214,14 +218,17 @@ class K3sMonitorService: # 取得 Telegram Gateway gateway = get_telegram_gateway() - if not gateway._initialized: - await gateway.initialize() # 發送訊息 formatted = status.format() - result = await gateway.send_text(formatted) + result = await gateway.send_text( + formatted, + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) - if result: + if _telegram_send_delivery_succeeded(result): logger.info("k3s_daily_report_sent", date=status.report_date) return True else: diff --git a/apps/api/src/services/notification_matrix.py b/apps/api/src/services/notification_matrix.py index fa6f2739e..921e23db2 100644 --- a/apps/api/src/services/notification_matrix.py +++ b/apps/api/src/services/notification_matrix.py @@ -1,64 +1,808 @@ -""" -Notification routing matrix — ADR-093 -====================================== -單一矩陣決定每種通知類型的發送目標，取代 telegram_gateway.py 內 24 處硬碼 chat_id。 +"""Fail-closed Telegram notification routing policy. -設計原則： -- 正式告警目的地一律為 SRE_GROUP_CHAT_ID -- SRE_GROUP_CHAT_ID 缺失時必須顯示配置缺口，不得旁路到個人或其他群組 -- 未知通知類型預設發群組 - -2026-04-25 ogt + Claude Sonnet 4.6 +The legacy ``TYPE-*`` matrix remains for existing callers, but an unknown type +no longer falls through to the shared SRE room. Product monitoring routes are +loaded from a committed, machine-readable registry. This module only returns +policy decisions: it never reads Telegram credentials or chat IDs, sends a +message, writes a receipt, or changes a runtime route. """ + from __future__ import annotations + +import json +import re from dataclasses import dataclass from enum import Enum +from pathlib import Path +from typing import Any + +from src.services.snapshot_paths import default_security_dir class Destination(str, Enum): - DM = "dm" # legacy alias: 2026-06-12 起不再旁路至 DM - GROUP = "group" # SRE_GROUP_CHAT_ID - BOTH = "both" # legacy alias: 2026-04-30 起視為 group-first + DM = "dm" # legacy alias; formal alerts do not fall back to DM + GROUP = "group" # symbolic shared group selected by the caller + BOTH = "both" # legacy alias; retained for import compatibility + BLOCKED = "blocked" @dataclass(frozen=True) class RoutingRule: destination: Destination - strip_buttons_for_group: bool = False # BOTH 時群組版是否去除 Callback Button + strip_buttons_for_group: bool = False -# ADR-093 D1-D4 路由矩陣 -# 2026-06-12 Codex: 所有正式告警只送 AwoooI SRE 戰情室；缺群組設定時回空清單。 +# ADR-093 legacy notification types. The global product routing registry below +# supersedes its unsafe "unknown -> shared group" default. NOTIFICATION_ROUTING: dict[str, RoutingRule] = { - "TYPE-1": RoutingRule(Destination.GROUP), - "TYPE-2": RoutingRule(Destination.GROUP), - "TYPE-3": RoutingRule(Destination.GROUP), - "TYPE-4": RoutingRule(Destination.GROUP), + "TYPE-1": RoutingRule(Destination.BLOCKED), + "TYPE-2": RoutingRule(Destination.GROUP), + "TYPE-3": RoutingRule(Destination.GROUP), + "TYPE-4": RoutingRule(Destination.GROUP), "TYPE-4D": RoutingRule(Destination.GROUP), "TYPE-5S": RoutingRule(Destination.GROUP), - "TYPE-6B": RoutingRule(Destination.GROUP), + "TYPE-6B": RoutingRule(Destination.BLOCKED), "TYPE-7E": RoutingRule(Destination.GROUP), "TYPE-8M": RoutingRule(Destination.GROUP), } -_DEFAULT_RULE = RoutingRule(Destination.GROUP) +_DEFAULT_RULE = RoutingRule(Destination.BLOCKED) +_LEGACY_CANONICAL_ROUTES: dict[str, tuple[str, str, str]] = { + "TYPE-2": ("awoooi", "incident_lifecycle", "P1"), + "TYPE-3": ("awoooi", "incident_lifecycle", "P1"), + "TYPE-4": ("awoooi", "incident_lifecycle", "P1"), + "TYPE-4D": ("awoooi", "security_incident", "P1"), + "TYPE-5S": ("awoooi", "security_incident", "P1"), + "TYPE-7E": ("awoooi", "incident_lifecycle", "P0"), + "TYPE-8M": ("awoooi", "shared_infrastructure", "P1"), +} +_SCHEMA_VERSION = "telegram_canonical_routing_registry_v1" +_REGISTRY_FILENAME = "telegram-canonical-routing-registry.snapshot.json" +_DEFAULT_SECURITY_DIR = default_security_dir(Path(__file__)) +_EXPECTED_PRODUCT_IDS = frozenset( + { + "2026fifa", + "agent-bounty-protocol", + "awooogo", + "awoooi", + "bitan-pharmacy", + "clawbot-openclaw", + "momo-pro", + "stockplatform-v2", + "tsenyang-website", + "vibework", + "vtuber", + } +) +_ALLOWED_SEVERITIES = frozenset({"P0", "P1", "P2", "P3"}) +_ALLOWED_EGRESS_STATUSES = frozenset( + {"allowed", "blocked", "disabled", "not_implemented"} +) +_SHARED_SRE_CHAT_ALIAS = "awoooi_sre_war_room" +_SHARED_MONITORING_BOT_ALIAS = "tsenyang_bot" +_SHARED_MONITORING_BOT_OWNER = "sre_shared_runtime" +_SHARED_SRE_SIGNAL_FAMILIES = frozenset( + { + "shared_infrastructure", + "security_incident", + "disaster_recovery", + "incident_lifecycle", + } +) +_REQUIRED_ROUTE_FIELDS = frozenset( + { + "route_id", + "product_id", + "signal_family", + "severity", + "scope", + "bot_alias", + "chat_alias", + "allowed_destination", + "receipt_backend", + "owner", + "egress_status", + } +) +_TOP_LEVEL_FIELDS = frozenset( + { + "schema_version", + "generated_at", + "status", + "mode", + "supersedes", + "policy", + "destination_roles", + "products", + "routes", + "rollups", + "execution_boundaries", + } +) +_SUPERSEDES_FIELDS = frozenset( + { + "policy_ref", + "superseded_behavior", + "superseded_by", + "owner", + "expiry", + "replacement", + "verifier", + } +) +_POLICY_FIELDS = frozenset( + { + "default_decision", + "unknown_product_decision", + "unknown_signal_family_decision", + "unknown_severity_decision", + "unknown_destination_decision", + "legacy_runtime_resolver", + "legacy_shared_sre_allowed_types", + "legacy_shared_sre_blocked_types", + "raw_chat_id_allowed", + "shared_sre_rule", + "shared_sre_forbidden", + } +) +_DESTINATION_ROLE_FIELDS = frozenset( + { + "visible_label", + "proven_runtime_alias", + "runtime_alias_evidence", + "chat_alias", + "bot_alias", + "bot_owner", + "monitoring_owner", + "role", + "monitoring_egress_policy", + "runtime_identifier_in_registry", + } +) +_PRODUCT_FIELDS = frozenset( + { + "product_id", + "display_name", + "owner", + "source_audit_status", + "egress_status", + } +) +_ROLLUP_FIELDS = frozenset( + { + "product_count", + "route_count", + "allowed_route_count", + "blocked_route_count", + "disabled_route_count", + "not_implemented_route_count", + "shared_sre_allowed_route_count", + "raw_numeric_destination_count", + "runtime_route_change_count", + "telegram_send_count", + } +) +_EXECUTION_BOUNDARY_FIELDS = frozenset( + { + "telegram_send_allowed", + "bot_api_call_allowed", + "secret_value_read_allowed", + "raw_chat_id_storage_allowed", + "runtime_route_change_allowed", + "production_deploy_allowed", + } +) +_SAFE_SENSITIVE_CONTROL_KEYS = frozenset( + { + "raw_chat_id_allowed", + "raw_chat_id_storage_allowed", + "secret_value_read_allowed", + } +) +_SENSITIVE_KEY_PATTERN = re.compile( + r"(?:^|_)(?:chat_id|bot_token|token|secret|secret_value|authorization(?:_header)?|" + r"webhook_url|api_key|password|credential|private_key|cookie|session)(?:$|_)", + re.IGNORECASE, +) +_RAW_NUMERIC_IDENTIFIER_PATTERN = re.compile(r"^-?\d{8,}$") +_BOT_TOKEN_VALUE_PATTERN = re.compile(r"\d{6,}:[A-Za-z0-9_-]{20,}") +_BOT_API_TOKEN_URL_PATTERN = re.compile(r"api\.telegram\.org/bot[^/\s]+", re.IGNORECASE) def get_routing_rule(notification_type: str) -> RoutingRule: - """根據通知類型回傳路由規則。未知類型預設發群組。""" + """Return a legacy route; unknown types are explicitly blocked.""" return NOTIFICATION_ROUTING.get(notification_type, _DEFAULT_RULE) +def get_legacy_canonical_route_context( + notification_type: str, +) -> dict[str, str] | None: + """Map an allowlisted legacy type to explicit canonical route dimensions.""" + canonical_route = _LEGACY_CANONICAL_ROUTES.get(notification_type) + if canonical_route is None: + return None + product_id, signal_family, severity = canonical_route + return { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + "legacy_notification_type": notification_type, + } + + def resolve_chat_ids( notification_type: str, dm_chat_id: str, group_chat_id: str, *, tg_group_cutover: bool = False, + registry: dict[str, Any] | None = None, ) -> list[str]: - """ - 回傳此通知應發送的 chat_id 清單。 - tg_group_cutover 僅保留為舊 caller 相容參數；正式策略永遠群組優先。 - """ - _ = get_routing_rule(notification_type) + """Resolve a legacy type through the canonical product policy.""" + _ = (dm_chat_id, tg_group_cutover) + rule = get_routing_rule(notification_type) + if rule.destination is not Destination.GROUP: + return [] + + canonical_route = get_legacy_canonical_route_context(notification_type) + if canonical_route is None: + return [] + try: + decision = resolve_canonical_telegram_route( + canonical_route["product_id"], + canonical_route["signal_family"], + canonical_route["severity"], + requested_destination=("telegram_chat_alias:awoooi_sre_war_room"), + registry=registry, + ) + except (FileNotFoundError, TypeError, ValueError, json.JSONDecodeError): + return [] + if decision["decision"] != "allow": + return [] return [group_chat_id] if group_chat_id else [] + + +def load_canonical_telegram_routing_registry( + security_dir: Path | None = None, +) -> dict[str, Any]: + """Load and validate the committed 11-product routing registry.""" + registry_path = (security_dir or _DEFAULT_SECURITY_DIR) / _REGISTRY_FILENAME + if not registry_path.is_file(): + raise FileNotFoundError( + f"canonical Telegram routing registry missing: {registry_path}" + ) + + with registry_path.open(encoding="utf-8") as handle: + payload = json.load(handle) + + if not isinstance(payload, dict): + raise ValueError(f"{registry_path}: expected JSON object") + _validate_canonical_registry(payload, str(registry_path)) + return payload + + +def resolve_canonical_telegram_route( + product_id: str, + signal_family: str, + severity: str, + *, + requested_destination: str | None = None, + registry: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Return one symbolic route decision, defaulting to a no-egress terminal.""" + payload = ( + load_canonical_telegram_routing_registry() if registry is None else registry + ) + _validate_canonical_registry(payload, "in-memory canonical Telegram registry") + + product = next( + ( + product + for product in payload["products"] + if product["product_id"] == product_id + ), + None, + ) + + normalized_severity = severity.strip().upper() + candidates = [ + route + for route in payload["routes"] + if route["product_id"] == product_id + and route["signal_family"] in {signal_family, "*"} + and normalized_severity in route["severity"] + ] + candidates.sort(key=lambda route: route["signal_family"] == "*", reverse=False) + + if not candidates: + return _blocked_decision( + product_id=product_id, + signal_family=signal_family, + severity=normalized_severity, + reason="unknown_product_signal_or_severity", + ) + + route = dict(candidates[0]) + route["decision"] = ( + "allow" + if product is not None + and product["egress_status"] == "allowed" + and route["egress_status"] == "allowed" + else "block" + ) + route["reason"] = route.get("blocked_reason") or "canonical_route_allowed" + + if ( + requested_destination is not None + and requested_destination != route["allowed_destination"] + ): + route.update( + { + "decision": "block", + "egress_status": "blocked", + "reason": "requested_destination_not_allowed", + "requested_destination": requested_destination, + } + ) + return route + + +def _blocked_decision( + *, product_id: str, signal_family: str, severity: str, reason: str +) -> dict[str, Any]: + return { + "route_id": None, + "product_id": product_id, + "signal_family": signal_family, + "severity": [severity], + "scope": "unknown", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "unassigned", + "egress_status": "blocked", + "decision": "block", + "reason": reason, + } + + +def _validate_canonical_registry(payload: dict[str, Any], label: str) -> None: + _require_exact_object_fields(payload, _TOP_LEVEL_FIELDS, label) + if payload.get("schema_version") != _SCHEMA_VERSION: + raise ValueError(f"{label}: expected schema_version={_SCHEMA_VERSION}") + if payload.get("status") != "source_policy_ready_runtime_not_applied": + raise ValueError(f"{label}: unsupported status") + if payload.get("mode") != "source_only_no_secret_no_telegram_send_no_route_change": + raise ValueError(f"{label}: unsupported mode") + if not isinstance(payload.get("generated_at"), str) or not payload["generated_at"]: + raise ValueError(f"{label}: generated_at must be a non-empty string") + + _validate_no_sensitive_runtime_material(payload, label) + + supersedes = payload["supersedes"] + _require_exact_object_fields( + supersedes, + _SUPERSEDES_FIELDS, + f"{label}.supersedes", + ) + for field in _SUPERSEDES_FIELDS: + if not isinstance(supersedes[field], str) or not supersedes[field].strip(): + raise ValueError(f"{label}.supersedes.{field} must be non-empty") + if supersedes["superseded_by"] != "global_product_governance_v2": + raise ValueError(f"{label}: superseded_by must be global governance v2") + + policy = payload["policy"] + _require_exact_object_fields(policy, _POLICY_FIELDS, f"{label}.policy") + for field in ( + "default_decision", + "unknown_product_decision", + "unknown_signal_family_decision", + "unknown_severity_decision", + "unknown_destination_decision", + ): + if policy[field] != "block": + raise ValueError(f"{label}.policy.{field} must be block") + if policy["raw_chat_id_allowed"] is not False: + raise ValueError(f"{label}.policy.raw_chat_id_allowed must be false") + expected_legacy_allowed = set(_LEGACY_CANONICAL_ROUTES) + legacy_allowed = policy["legacy_shared_sre_allowed_types"] + legacy_blocked = policy["legacy_shared_sre_blocked_types"] + shared_forbidden = policy["shared_sre_forbidden"] + if ( + not isinstance(legacy_allowed, list) + or len(legacy_allowed) != len(set(legacy_allowed)) + or set(legacy_allowed) != expected_legacy_allowed + ): + raise ValueError(f"{label}: legacy allowed types drifted") + if ( + not isinstance(legacy_blocked, list) + or len(legacy_blocked) != len(set(legacy_blocked)) + or set(legacy_blocked) != {"TYPE-1", "TYPE-6B", "UNKNOWN"} + ): + raise ValueError(f"{label}: legacy blocked types drifted") + if not isinstance(shared_forbidden, list) or not shared_forbidden or any( + not isinstance(item, str) or not item.strip() for item in shared_forbidden + ): + raise ValueError(f"{label}: shared_sre_forbidden must be non-empty strings") + for field in ("legacy_runtime_resolver", "shared_sre_rule"): + if not isinstance(policy[field], str) or not policy[field].strip(): + raise ValueError(f"{label}.policy.{field} must be non-empty") + + boundaries = payload["execution_boundaries"] + _require_exact_object_fields( + boundaries, + _EXECUTION_BOUNDARY_FIELDS, + f"{label}.execution_boundaries", + ) + unsafe = sorted( + key for key in _EXECUTION_BOUNDARY_FIELDS if boundaries[key] is not False + ) + if unsafe: + raise ValueError(f"{label}: execution boundaries must remain false: {unsafe}") + + destination_roles = payload["destination_roles"] + if not isinstance(destination_roles, list) or not destination_roles: + raise ValueError(f"{label}: destination_roles must be a non-empty array") + observed_role_bindings: set[tuple[str, str]] = set() + for index, role in enumerate(destination_roles): + role_label = f"{label}.destination_roles[{index}]" + _require_exact_object_fields(role, _DESTINATION_ROLE_FIELDS, role_label) + for field in ( + "visible_label", + "proven_runtime_alias", + "runtime_alias_evidence", + "chat_alias", + "bot_alias", + "bot_owner", + "role", + "monitoring_egress_policy", + ): + if not isinstance(role[field], str) or not role[field].strip(): + raise ValueError(f"{role_label}.{field} must be non-empty") + if role["runtime_identifier_in_registry"] is not False: + raise ValueError( + f"{role_label}.runtime_identifier_in_registry must be false" + ) + if not isinstance(role["monitoring_owner"], bool): + raise ValueError(f"{role_label}.monitoring_owner must be boolean") + if role["runtime_alias_evidence"] == "visible_label_only": + if role["proven_runtime_alias"] != "none": + raise ValueError( + f"{role_label}: visible label must not claim a proven runtime alias" + ) + elif role["runtime_alias_evidence"] == "proven_runtime_receipt": + if role["proven_runtime_alias"] == "none": + raise ValueError( + f"{role_label}: proven runtime receipt requires an alias" + ) + elif role["runtime_alias_evidence"] not in { + "source_config_only", + "configured_unreachable", + }: + raise ValueError(f"{role_label}: unsupported runtime_alias_evidence") + for field in ("chat_alias", "bot_alias"): + if _RAW_NUMERIC_IDENTIFIER_PATTERN.fullmatch(role[field]): + raise ValueError(f"{role_label}: raw numeric runtime identifier") + binding = (role["chat_alias"], role["bot_alias"]) + if binding in observed_role_bindings: + raise ValueError(f"{role_label}: duplicate symbolic destination binding") + observed_role_bindings.add(binding) + + products = payload["products"] + routes = payload["routes"] + if not isinstance(products, list) or not isinstance(routes, list): + raise ValueError(f"{label}: products and routes must be arrays") + + for index, product in enumerate(products): + product_label = f"{label}.products[{index}]" + _require_exact_object_fields(product, _PRODUCT_FIELDS, product_label) + for field in ("product_id", "display_name", "owner", "source_audit_status"): + if not isinstance(product[field], str) or not product[field].strip(): + raise ValueError(f"{product_label}.{field} must be non-empty") + if product["egress_status"] not in _ALLOWED_EGRESS_STATUSES: + raise ValueError(f"{product_label}: unsupported product egress_status") + product_ids = [product["product_id"] for product in products] + if len(product_ids) != len(set(product_ids)): + raise ValueError(f"{label}: duplicate product_id") + if set(product_ids) != _EXPECTED_PRODUCT_IDS: + raise ValueError( + f"{label}: product registry must cover the canonical 11 products" + ) + + route_ids: set[str] = set() + covered_products: set[str] = set() + for index, route in enumerate(routes): + route_label = f"{label}.routes[{index}]" + _require_exact_object_fields( + route, + _REQUIRED_ROUTE_FIELDS, + route_label, + optional_fields=frozenset({"blocked_reason"}), + ) + + route_id = str(route["route_id"]) + if not route_id or route_id in route_ids: + raise ValueError( + f"{label}: route_id must be non-empty and unique: {route_id!r}" + ) + route_ids.add(route_id) + + product_id = str(route["product_id"]) + if product_id not in _EXPECTED_PRODUCT_IDS: + raise ValueError(f"{label}: unknown product_id in route: {product_id}") + covered_products.add(product_id) + + severity = route["severity"] + if ( + not isinstance(severity, list) + or not severity + or len(severity) != len(set(severity)) + ): + raise ValueError(f"{label}: {route_id} severity must be a non-empty array") + if not set(severity).issubset(_ALLOWED_SEVERITIES): + raise ValueError(f"{label}: {route_id} has unsupported severity") + + egress_status = str(route["egress_status"]) + if egress_status not in _ALLOWED_EGRESS_STATUSES: + raise ValueError(f"{label}: {route_id} has unsupported egress_status") + if route["scope"] not in {"shared", "product"}: + raise ValueError(f"{label}: {route_id} has unsupported scope") + for field in ( + "route_id", + "product_id", + "signal_family", + "scope", + "bot_alias", + "chat_alias", + "allowed_destination", + "receipt_backend", + "owner", + "egress_status", + ): + if not isinstance(route[field], str) or not route[field].strip(): + raise ValueError(f"{route_label}.{field} must be non-empty") + + _validate_no_runtime_identifiers(route, label, route_id) + _validate_route_destination(route, label, route_id) + _validate_shared_sre_boundary(route, label, route_id) + + if covered_products != _EXPECTED_PRODUCT_IDS: + raise ValueError( + f"{label}: every canonical product requires at least one route" + ) + + by_product = {product["product_id"]: product for product in products} + for product_id, product in by_product.items(): + expected_status = product.get("egress_status") + if expected_status not in _ALLOWED_EGRESS_STATUSES: + raise ValueError( + f"{label}: {product_id} has unsupported product egress_status" + ) + product_routes = [ + route for route in routes if route["product_id"] == product_id + ] + allowed_route_count = sum( + route["egress_status"] == "allowed" for route in product_routes + ) + if expected_status != "allowed" and allowed_route_count: + raise ValueError(f"{label}: {product_id} must not have an allowed route") + if expected_status == "allowed" and allowed_route_count == 0: + raise ValueError( + f"{label}: {product_id} requires at least one allowed route" + ) + if expected_status in {"disabled", "not_implemented"} and any( + route["egress_status"] != expected_status for route in product_routes + ): + raise ValueError( + f"{label}: {product_id} route status must match product disposition" + ) + + # Ownership is a cross-object invariant. Validate it after product-level + # disposition so a disabled product cannot be made allowed merely by also + # forging a destination-role binding. + for route in routes: + _validate_route_role_ownership( + route, + destination_roles, + label, + str(route["route_id"]), + ) + + _validate_registry_rollups(payload, label) + + +def _require_exact_object_fields( + value: object, + required_fields: frozenset[str], + label: str, + *, + optional_fields: frozenset[str] = frozenset(), +) -> None: + if not isinstance(value, dict): + raise ValueError(f"{label}: expected JSON object") + missing = sorted(required_fields - value.keys()) + unexpected = sorted(value.keys() - required_fields - optional_fields) + if missing: + raise ValueError(f"{label}: missing required fields: {missing}") + if unexpected: + raise ValueError(f"{label}: unexpected fields: {unexpected}") + + +def _validate_no_runtime_identifiers( + route: dict[str, Any], label: str, route_id: str +) -> None: + forbidden_fields = {"chat_id", "bot_token", "token", "secret", "webhook_url"} + present = sorted(forbidden_fields & route.keys()) + if present: + raise ValueError( + f"{label}: {route_id} contains forbidden runtime fields: {present}" + ) + + for field in ("bot_alias", "chat_alias", "allowed_destination"): + value = str(route[field]) + if value.lstrip("-").isdigit(): + raise ValueError( + f"{label}: {route_id} must not contain a raw Telegram identifier" + ) + + +def _validate_no_sensitive_runtime_material(payload: Any, label: str) -> None: + """Reject secret-shaped keys/values and raw numeric identifiers recursively.""" + + def walk(value: Any, path: str) -> None: + if isinstance(value, dict): + for raw_key, nested in value.items(): + key = str(raw_key) + child_path = f"{path}.{key}" + if _SENSITIVE_KEY_PATTERN.search(key): + safe_control = ( + key in _SAFE_SENSITIVE_CONTROL_KEYS and nested is False + ) + if not safe_control: + raise ValueError( + f"{label}: forbidden sensitive runtime key at {child_path}" + ) + walk(nested, child_path) + return + if isinstance(value, list): + for index, nested in enumerate(value): + walk(nested, f"{path}[{index}]") + return + if isinstance(value, bool) or value is None: + return + if isinstance(value, int): + if abs(value) >= 1_000_000_000: + raise ValueError(f"{label}: raw numeric runtime identifier at {path}") + return + if not isinstance(value, str): + return + if _RAW_NUMERIC_IDENTIFIER_PATTERN.fullmatch(value): + raise ValueError(f"{label}: raw numeric runtime identifier at {path}") + if _BOT_TOKEN_VALUE_PATTERN.search(value) or _BOT_API_TOKEN_URL_PATTERN.search( + value + ): + raise ValueError(f"{label}: secret-shaped runtime value at {path}") + + walk(payload, "registry") + + +def _validate_route_destination( + route: dict[str, Any], label: str, route_id: str +) -> None: + status = route["egress_status"] + destination = route["allowed_destination"] + if status == "allowed": + expected = f"telegram_chat_alias:{route['chat_alias']}" + if destination != expected: + raise ValueError( + f"{label}: {route_id} allowed destination must match chat_alias" + ) + if route["bot_alias"] == "none" or route["chat_alias"] == "none": + raise ValueError( + f"{label}: {route_id} allowed route requires symbolic aliases" + ) + if route["receipt_backend"] == "none": + raise ValueError( + f"{label}: {route_id} allowed route requires a receipt backend" + ) + else: + if destination != "blocked": + raise ValueError(f"{label}: {route_id} non-allowed route must be blocked") + if not route.get("blocked_reason"): + raise ValueError( + f"{label}: {route_id} non-allowed route requires blocked_reason" + ) + + +def _validate_shared_sre_boundary( + route: dict[str, Any], label: str, route_id: str +) -> None: + if route["chat_alias"] != _SHARED_SRE_CHAT_ALIAS: + return + invalid = ( + route["product_id"] != "awoooi" + or route["scope"] != "shared" + or route["signal_family"] not in _SHARED_SRE_SIGNAL_FAMILIES + or not set(route["severity"]).issubset({"P0", "P1"}) + or route["bot_alias"] != _SHARED_MONITORING_BOT_ALIAS + ) + if invalid: + raise ValueError( + f"{label}: {route_id} violates shared SRE P0/P1 infra/security/DR/lifecycle boundary" + ) + + +def _validate_route_role_ownership( + route: dict[str, Any], + destination_roles: list[dict[str, Any]], + label: str, + route_id: str, +) -> None: + """Cross-check every allowed route against its declared destination owner.""" + if route["egress_status"] != "allowed": + return + role = next( + ( + item + for item in destination_roles + if item["chat_alias"] == route["chat_alias"] + and item["bot_alias"] == route["bot_alias"] + ), + None, + ) + if role is None: + raise ValueError( + f"{label}: {route_id} has no matching destination_roles ownership binding" + ) + if role["monitoring_egress_policy"] != "allowlist_only": + raise ValueError( + f"{label}: {route_id} destination role is not monitoring allowlisted" + ) + if role["monitoring_owner"] is not True: + raise ValueError( + f"{label}: {route_id} destination role is not the monitoring owner" + ) + if role["bot_owner"] != _SHARED_MONITORING_BOT_OWNER: + raise ValueError( + f"{label}: {route_id} bot ownership does not match shared monitoring policy" + ) + + +def _validate_registry_rollups(payload: dict[str, Any], label: str) -> None: + rollups = payload["rollups"] + _require_exact_object_fields( + rollups, + _ROLLUP_FIELDS, + f"{label}.rollups", + ) + routes = payload["routes"] + expected = { + "product_count": len(payload["products"]), + "route_count": len(routes), + "allowed_route_count": sum( + route["egress_status"] == "allowed" for route in routes + ), + "blocked_route_count": sum( + route["egress_status"] == "blocked" for route in routes + ), + "disabled_route_count": sum( + route["egress_status"] == "disabled" for route in routes + ), + "not_implemented_route_count": sum( + route["egress_status"] == "not_implemented" for route in routes + ), + "shared_sre_allowed_route_count": sum( + route["egress_status"] == "allowed" + and route["chat_alias"] == _SHARED_SRE_CHAT_ALIAS + for route in routes + ), + "raw_numeric_destination_count": 0, + "runtime_route_change_count": 0, + "telegram_send_count": 0, + } + drifted = sorted( + key for key, value in expected.items() if rollups.get(key) != value + ) + if drifted: + raise ValueError(f"{label}: registry rollup mismatch: {drifted}") diff --git a/apps/api/src/services/notifications/telegram.py b/apps/api/src/services/notifications/telegram.py index 4a21673ab..5eb7bb94b 100644 --- a/apps/api/src/services/notifications/telegram.py +++ b/apps/api/src/services/notifications/telegram.py @@ -9,8 +9,8 @@ from src.core.config import settings from src.core.logging import get_logger + from .base import ( - ExecutionStatus, NotificationMessage, NotificationProvider, NotificationResult, @@ -60,16 +60,54 @@ class TelegramWebhookProvider(NotificationProvider): message="Telegram bot token or chat_id not configured", ) try: - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() text = self._format(message) - resp = await gateway.send_alert_notification(text=text, parse_mode="HTML") + resp = await gateway.send_alert_notification( + text=text, + parse_mode="HTML", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + response_data = resp if isinstance(resp, dict) else None + if not _telegram_send_delivery_succeeded(resp): + delivery_status = ( + str(resp.get("_awooop_delivery_status") or "") + if isinstance(resp, dict) + else "" + ) + provider_send_performed = ( + resp.get("_awooop_provider_send_performed") + if isinstance(resp, dict) + else None + ) + if ( + delivery_status == "blocked_no_egress" + or provider_send_performed is False + ): + return NotificationResult( + status=NotificationStatus.SKIPPED, + provider=self.name, + message="Telegram notification blocked before provider send", + response_data=response_data, + ) + return NotificationResult( + status=NotificationStatus.FAILED, + provider=self.name, + message="Telegram provider did not acknowledge delivery", + response_data=response_data, + error="delivery_not_acknowledged", + ) return NotificationResult( status=NotificationStatus.SUCCESS, provider=self.name, message="Telegram notification sent", - response_data=resp if isinstance(resp, dict) else None, + response_data=response_data, ) except Exception as e: logger.exception("telegram_notification_exception", error=str(e)) @@ -88,8 +126,13 @@ class TelegramWebhookProvider(NotificationProvider): from src.services.telegram_gateway import get_telegram_gateway gw = get_telegram_gateway() - await gw.send_alert_notification(text="🔔 AWOOOI Telegram provider 連線測試") - return True + result = await gw.send_alert_notification( + text="AWOOOI Telegram provider no-egress policy probe", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) + return bool(result.get("ok")) except Exception as e: logger.error("telegram_connection_test_failed", error=str(e)) return False diff --git a/apps/api/src/services/post_execution_verifier.py b/apps/api/src/services/post_execution_verifier.py index 6c842e97b..e44e24e91 100644 --- a/apps/api/src/services/post_execution_verifier.py +++ b/apps/api/src/services/post_execution_verifier.py @@ -410,9 +410,10 @@ async def _send_rollback_proposal_alert( 不自動執行 rollback，僅通知人工評估。 """ try: - from src.core.config import get_settings - from src.services.telegram_gateway import get_telegram_gateway - _settings = get_settings() + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() regressions = assessment.get("regressions", []) @@ -429,20 +430,25 @@ async def _send_rollback_proposal_alert( f"此為提案，不會自動執行 Rollback" ) - target_chat_id = _settings.SRE_GROUP_CHAT_ID - await gateway._send_request( - "sendMessage", - { - "chat_id": target_chat_id, - "text": text, - "parse_mode": "HTML", - }, - ) - logger.info( - "rollback_proposal_sent", - incident_id=incident_id, - score=score, + delivery = await gateway.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + parse_mode="HTML", ) + if _telegram_send_delivery_succeeded(delivery): + logger.info( + "rollback_proposal_sent", + incident_id=incident_id, + score=score, + ) + else: + logger.warning( + "rollback_proposal_blocked_no_send", + incident_id=incident_id, + score=score, + ) except Exception: logger.warning( "rollback_proposal_send_failed", diff --git a/apps/api/src/services/report_generation_service.py b/apps/api/src/services/report_generation_service.py index 4e2b1f5a9..a87c46cec 100644 --- a/apps/api/src/services/report_generation_service.py +++ b/apps/api/src/services/report_generation_service.py @@ -569,38 +569,114 @@ class ReportGenerationService: Graceful Degradation: 失敗只記錄 log，不拋出例外 """ try: + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) + + gateway = get_telegram_gateway() + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ): + logger.warning( + "daily_report_no_send", + reason="canonical_route_unavailable", + ) + return + kpi = await self.collect_daily_kpi() source_health = await self.collect_report_source_health(days=1) report_text = self.format_daily_report(kpi, source_health) - from src.services.telegram_gateway import get_telegram_gateway - gateway = get_telegram_gateway() - await gateway.send_to_group(report_text, parse_mode="HTML") - - logger.info( - "daily_report_sent", - total_alerts=kpi.total_alerts, - auto_repair_rate=f"{kpi.auto_repair_rate:.1%}", + result = await gateway.send_to_group( + report_text, + parse_mode="HTML", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) + + if _telegram_send_delivery_succeeded(result): + logger.info( + "daily_report_sent", + total_alerts=kpi.total_alerts, + delivery_status=result.get("_awooop_delivery_status", "sent"), + auto_repair_rate=f"{kpi.auto_repair_rate:.1%}", + ) + else: + logger.warning( + "daily_report_no_send", + total_alerts=kpi.total_alerts, + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) except Exception as e: logger.error("daily_report_failed", error=str(e)) async def send_monthly_report(self) -> bool: """收集月報資料 → 組裝 → 推送 Telegram SRE 群組。""" try: + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) + + gateway = get_telegram_gateway() + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ): + logger.warning( + "monthly_report_no_send", + reason="canonical_route_unavailable", + ) + return False + source_health = await self.collect_report_source_health(days=30) report_text = self.format_monthly_report(source_health) - from src.services.telegram_gateway import get_telegram_gateway - gateway = get_telegram_gateway() - await gateway.send_to_group(report_text, parse_mode="HTML") - - logger.info( - "monthly_report_sent", - source_ok=(source_health.get("rollups") or {}).get("source_ok_count"), - source_total=(source_health.get("rollups") or {}).get("source_count"), + result = await gateway.send_to_group( + report_text, + parse_mode="HTML", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) - return True + + if _telegram_send_delivery_succeeded(result): + logger.info( + "monthly_report_sent", + source_ok=(source_health.get("rollups") or {}).get( + "source_ok_count" + ), + delivery_status=result.get("_awooop_delivery_status", "sent"), + source_total=(source_health.get("rollups") or {}).get( + "source_count" + ), + ) + return True + + logger.warning( + "monthly_report_no_send", + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) + return False except Exception as e: logger.error("monthly_report_failed", error=str(e)) return False @@ -663,7 +739,10 @@ class ReportGenerationService: import asyncio as _asyncio report_text = self.format_postmortem(data) await self._persist_postmortem_km(data, report_text) - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() max_attempts = 3 @@ -671,7 +750,22 @@ class ReportGenerationService: last_error: Exception | None = None for attempt in range(1, max_attempts + 1): try: - await gateway.send_to_group(report_text, parse_mode="HTML") + result = await gateway.send_to_group( + report_text, + parse_mode="HTML", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + if not _telegram_send_delivery_succeeded(result): + delivery_status = ( + result.get("_awooop_delivery_status") + if isinstance(result, dict) + else None + ) + raise RuntimeError( + str(delivery_status or "missing_provider_ack") + ) logger.info( "postmortem_sent", incident_id=incident_id, @@ -705,7 +799,23 @@ class ReportGenerationService: f"Duration: {duration_minutes:.1f} 分鐘\n" f"Error: {str(last_error)[:200]}" ) - await gateway.send_to_group(fallback_text, parse_mode="HTML") + fallback_result = await gateway.send_to_group( + fallback_text, + parse_mode="HTML", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + if not _telegram_send_delivery_succeeded(fallback_result): + logger.error( + "postmortem_fallback_no_send", + incident_id=incident_id, + delivery_status=( + fallback_result.get("_awooop_delivery_status") + if isinstance(fallback_result, dict) + else "missing_provider_ack" + ), + ) except Exception as _fe: logger.error( "postmortem_fallback_failed", diff --git a/apps/api/src/services/runbook_generator.py b/apps/api/src/services/runbook_generator.py index 565b20536..5b1d47ed1 100644 --- a/apps/api/src/services/runbook_generator.py +++ b/apps/api/src/services/runbook_generator.py @@ -389,7 +389,12 @@ class NemotronRunbookGenerator: try: from src.services.telegram_gateway import get_telegram_gateway tg = get_telegram_gateway() - await tg.send_text(format_runbook_review_card(incident, entry_id, content_preview)) + await tg.send_text( + format_runbook_review_card(incident, entry_id, content_preview), + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) except Exception as e: logger.warning("runbook_review_card_failed", error=str(e)) @@ -406,7 +411,10 @@ class NemotronRunbookGenerator: f"⚠️ 已記錄失敗案例\n" f"Incident: {incident.incident_id}\n" f"標題: {title}\n\n" - f"相同症狀的後續告警將阻斷自動修復，要求人工介入。" + f"相同症狀的後續告警將阻斷自動修復，要求人工介入。", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", ) except Exception as e: logger.warning("anti_pattern_notification_failed", error=str(e)) diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 9b3dd07f8..00e382f00 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -28,6 +28,7 @@ import html import json import os import re +from collections.abc import Mapping from dataclasses import dataclass, field from datetime import UTC, datetime from uuid import NAMESPACE_URL, UUID, uuid5 @@ -60,6 +61,10 @@ from src.services.backup_restore_signal_automation import ( build_backup_restore_signal_automation_contract, ) from src.services.chat_manager import get_chat_manager +from src.services.notification_matrix import ( + get_legacy_canonical_route_context, + resolve_canonical_telegram_route, +) from src.services.operator_outcome import ( build_operator_outcome, normalize_operator_outcome, @@ -138,6 +143,17 @@ _AI_ADVISORY_CALLBACK_RE = re.compile( _CODE_REF_RE = re.compile(r"([0-9a-f]{7,12})", re.IGNORECASE) _TELEGRAM_HTML_CHUNK_LIMIT = 3600 _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY = "_awooop_source_envelope_extra" +_CANONICAL_ROUTE_CONTEXT_KEY = "_awoooi_canonical_telegram_route" +_LEGACY_NOTIFICATION_TYPE_KEY = "_awoooi_legacy_notification_type" +_NON_MONITORING_INTERACTION_KEY = "_awoooi_non_monitoring_interaction" +_CANONICAL_SRE_DESTINATION = "telegram_chat_alias:awoooi_sre_war_room" +_NON_MONITORING_INTERACTION_KINDS = frozenset( + { + "operator_command_reply", + "callback_reply", + "bot_discussion_reply", + } +) _CONTROLLED_APPLY_OUTBOUND_FINALIZE_ATTEMPTS = 3 _NO_WRITE_REPLAY_DIGEST_WINDOW_MINUTES = 30 _HOST_RESOURCE_ALERT_HEADER_RE = re.compile( @@ -825,6 +841,32 @@ def normalize_telegram_send_message_payload(method: str, payload: dict) -> dict: return normalized_payload +def _telegram_send_delivery_succeeded(result: object) -> bool: + """Require affirmative provider delivery evidence from ``_send_request``. + + Final-exit policy blocks are returned as structured receipts rather than + exceptions. Callers that implement bounded retry must therefore reject an + explicit no-send flag or any non-terminal delivery status even when + ``ok`` was accidentally left truthy by an adapter. + """ + if not isinstance(result, Mapping) or result.get("ok") is not True: + return False + + delivery_status = str(result.get("_awooop_delivery_status") or "").strip() + if delivery_status and delivery_status not in {"sent", "sent_reused"}: + return False + if result.get("_awooop_provider_send_performed") is False: + return False + + route_receipt = result.get("_awoooi_canonical_route_receipt") + if ( + isinstance(route_receipt, Mapping) + and route_receipt.get("provider_send_performed") is False + ): + return False + return True + + def _top_gateway_bucket( buckets: list[dict[str, object]], field: str, @@ -2777,6 +2819,7 @@ def _callback_reply_source_envelope_extra( callback_action: str | None = None, parse_mode: str | None = None, error: str | None = None, + parent_message_id: int | None = None, km_stale_completion_summary: dict[str, object] | None = None, awooop_status_chain: dict[str, object] | None = None, ) -> dict[str, object] | None: @@ -2798,6 +2841,8 @@ def _callback_reply_source_envelope_extra( callback_reply["parse_mode"] = parse_mode if error: callback_reply["error"] = _sanitize_telegram_error(error)[:300] + if parent_message_id is not None and parent_message_id > 0: + callback_reply["parent_provider_message_id"] = str(parent_message_id) extra: dict[str, object] = { "callback_reply": callback_reply, @@ -2805,6 +2850,10 @@ def _callback_reply_source_envelope_extra( "incident_ids": [incident_id], }, } + if parent_message_id is not None and parent_message_id > 0: + extra["source_refs"]["parent_provider_message_ids"] = [ + str(parent_message_id) + ] km_snapshot = _callback_reply_km_stale_completion_snapshot( km_stale_completion_summary, ) @@ -2816,6 +2865,42 @@ def _callback_reply_source_envelope_extra( return extra +def _incident_lifecycle_parent_source_envelope_extra( + *, + incident_id: str | None = None, + approval_id: str | None = None, + parent_lookup_key: str, + parent_message_id: int, + event_kind: str, +) -> dict[str, object]: + """Persist the canonical lifecycle thread parent in the outbound mirror. + + Lifecycle updates are autonomous monitoring events, not replies to a newly + verified inbound operator interaction. Their Redis-backed parent lookup is + therefore recorded in the durable Channel Hub source envelope while the + send itself remains governed by the canonical incident lifecycle route. + """ + parent_id = str(parent_message_id) + correlation_id = str(incident_id or approval_id or "") + correlation_kind = "incident" if incident_id else "approval" + source_ref_key = "incident_ids" if incident_id else "approval_ids" + return { + "incident_lifecycle_parent_receipt": { + "schema_version": "telegram_incident_lifecycle_parent_receipt_v1", + "correlation_id": correlation_id, + "correlation_kind": correlation_kind, + "event_kind": event_kind, + "parent_provider_message_id": parent_id, + "parent_lookup_key": parent_lookup_key, + "durable_mirror_required": True, + }, + "source_refs": { + source_ref_key: [correlation_id], + "parent_provider_message_ids": [parent_id], + }, + } + + def _alert_notification_source_envelope_extra( *, safe_text: str, @@ -3369,6 +3454,14 @@ def _merge_outbound_source_envelope_extra( if isinstance(notification_policy, dict): envelope["notification_policy"] = notification_policy + incident_lifecycle_parent_receipt = extra.get( + "incident_lifecycle_parent_receipt" + ) + if isinstance(incident_lifecycle_parent_receipt, dict): + envelope["incident_lifecycle_parent_receipt"] = ( + incident_lifecycle_parent_receipt + ) + extra_refs = extra.get("source_refs") if isinstance(extra_refs, dict): source_refs = envelope.setdefault("source_refs", {}) @@ -3385,6 +3478,41 @@ def _merge_outbound_source_envelope_extra( return envelope + +def _acknowledge_callback_reply_source_envelope( + source_envelope_extra: dict[str, object] | None, + *, + delivery_result: object, + provider_message_id: str, +) -> dict[str, object] | None: + """Promote callback attempt metadata only after affirmative provider ack.""" + if ( + not isinstance(source_envelope_extra, dict) + or not provider_message_id + or not _telegram_send_delivery_succeeded(delivery_result) + ): + return source_envelope_extra + + callback_reply = source_envelope_extra.get("callback_reply") + if not isinstance(callback_reply, dict): + return source_envelope_extra + + terminal_status = { + "callback_reply_attempt": "callback_reply_sent", + "callback_reply_fallback_attempt": "callback_reply_fallback_sent", + "callback_reply_rescue_attempt": "callback_reply_rescue_sent", + }.get(str(callback_reply.get("status") or "")) + if terminal_status is None: + return source_envelope_extra + + acknowledged_extra = dict(source_envelope_extra) + acknowledged_callback = dict(callback_reply) + acknowledged_callback["status"] = terminal_status + acknowledged_callback["provider_delivery_acknowledged"] = True + acknowledged_callback["provider_message_id"] = provider_message_id + acknowledged_extra["callback_reply"] = acknowledged_callback + return acknowledged_extra + # 2026-04-27 Claude Sonnet 4.6: B3 — LLM 動態 Telegram 按鈕 Feature Flag # true → 優先使用 ActionPlan.recommended_actions 動態生成按鈕 # false → 維持現有 callback_action_spec.yaml 路徑（預設，向下相容） @@ -5083,6 +5211,12 @@ class TelegramGateway: self._last_message_time: datetime | None = None self._heartbeat_task: asyncio.Task | None = None self._heartbeat_active = False + # Process-local mirrors of durable, provider-acknowledged dedup receipts. + # They avoid a second Redis read on repeated calls while preserving the + # rule that a blocked/no-egress result must never consume a dedup slot. + self._delivered_incident_update_dedup: set[str] = set() + self._delivered_global_failure_dedup: set[str] = set() + self._delivered_grouped_digest_dedup: set[str] = set() async def initialize(self) -> bool: """初始化 Gateway""" @@ -5115,14 +5249,223 @@ class TelegramGateway: @property def chat_id(self) -> str: - """取得正式產品告警 Chat ID。""" + """Runtime binding retained for edit/delete and inbound reply continuity.""" return settings.SRE_GROUP_CHAT_ID @property def alert_chat_id(self) -> str: - """告警訊息收件人：正式產品告警只送 AwoooI SRE 戰情室。""" + """Runtime binding; new monitoring sends are still policy-gated.""" return settings.SRE_GROUP_CHAT_ID + @staticmethod + def _blocked_route_receipt( + classifier: str, + *, + route_context: Mapping[str, object] | None = None, + ) -> dict[str, object]: + context = route_context or {} + return { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "blocked_no_egress", + "classifier": classifier, + "product_id": str(context.get("product_id") or "unknown")[:80], + "signal_family": str(context.get("signal_family") or "unknown")[:80], + "severity": str(context.get("severity") or "unknown")[:16], + "destination_alias": "none", + "provider_send_performed": False, + } + + def _authorize_canonical_send_message( + self, + *, + payload: Mapping[str, object], + route_context: object, + legacy_notification_type: object, + actual_bot_alias: str, + ) -> tuple[str | None, dict[str, object]]: + context: Mapping[str, object] | None = ( + route_context if isinstance(route_context, Mapping) else None + ) + legacy_type = str(legacy_notification_type or "").strip() + if legacy_type: + legacy_context = get_legacy_canonical_route_context(legacy_type) + if legacy_context is None: + return None, self._blocked_route_receipt( + "legacy_notification_type_not_allowed", + route_context={"signal_family": legacy_type}, + ) + context = legacy_context + + if context is None: + return None, self._blocked_route_receipt( + "missing_product_signal_severity_context" + ) + + product_id = str(context.get("product_id") or "").strip() + signal_family = str(context.get("signal_family") or "").strip() + severity = str(context.get("severity") or "").strip().upper() + normalized_context = { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } + if not product_id or not signal_family or not severity: + return None, self._blocked_route_receipt( + "incomplete_product_signal_severity_context", + route_context=normalized_context, + ) + + try: + decision = resolve_canonical_telegram_route( + product_id, + signal_family, + severity, + requested_destination=_CANONICAL_SRE_DESTINATION, + ) + except (FileNotFoundError, TypeError, ValueError, json.JSONDecodeError): + return None, self._blocked_route_receipt( + "canonical_registry_invalid_or_missing", + route_context=normalized_context, + ) + + if decision.get("decision") != "allow": + return None, self._blocked_route_receipt( + f"canonical_policy_blocked:{decision.get('reason') or 'unknown'}", + route_context=normalized_context, + ) + if decision.get("allowed_destination") != _CANONICAL_SRE_DESTINATION: + return None, self._blocked_route_receipt( + "canonical_destination_not_supported", + route_context=normalized_context, + ) + if decision.get("bot_alias") != actual_bot_alias: + return None, self._blocked_route_receipt( + "sender_bot_alias_not_authorized_for_route", + route_context=normalized_context, + ) + + canonical_chat_id = str(settings.SRE_GROUP_CHAT_ID or "") + if not canonical_chat_id: + return None, self._blocked_route_receipt( + "canonical_destination_unconfigured", + route_context=normalized_context, + ) + caller_chat_id = str(payload.get("chat_id") or "") + if caller_chat_id and caller_chat_id != canonical_chat_id: + return None, self._blocked_route_receipt( + "caller_destination_mismatch", + route_context=normalized_context, + ) + + return canonical_chat_id, { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "classifier": "canonical_shared_route_allowed", + **normalized_context, + "destination_alias": "awoooi_sre_war_room", + "sender_bot_alias": actual_bot_alias, + "provider_send_performed": False, + } + + def _authorize_non_monitoring_interaction( + self, + *, + payload: Mapping[str, object], + interaction_context: object, + actual_bot_alias: str, + ) -> tuple[str | None, dict[str, object]]: + context = ( + interaction_context + if isinstance(interaction_context, Mapping) + else None + ) + interaction_kind = str( + (context or {}).get("interaction_kind") or "" + ).strip() + if interaction_kind not in _NON_MONITORING_INTERACTION_KINDS: + return None, self._blocked_route_receipt( + "unknown_non_monitoring_interaction_kind", + route_context={"signal_family": interaction_kind or "unknown"}, + ) + + target_chat_id = str(payload.get("chat_id") or "") + if not target_chat_id: + return None, self._blocked_route_receipt( + "interaction_reply_target_missing", + route_context={"signal_family": interaction_kind}, + ) + + inbound_chat_id = str((context or {}).get("inbound_chat_id") or "") + inbound_message_raw = (context or {}).get("inbound_message_id") + try: + inbound_message_id = int(str(inbound_message_raw)) + except (TypeError, ValueError): + inbound_message_id = 0 + if not inbound_chat_id or inbound_message_id <= 0: + return None, self._blocked_route_receipt( + "interaction_inbound_context_missing", + route_context={"signal_family": interaction_kind}, + ) + if inbound_chat_id != target_chat_id: + return None, self._blocked_route_receipt( + "interaction_inbound_chat_mismatch", + route_context={"signal_family": interaction_kind}, + ) + + reply_parameters = payload.get("reply_parameters") + reply_message_raw = payload.get("reply_to_message_id") + if isinstance(reply_parameters, Mapping): + reply_message_raw = reply_parameters.get("message_id") + try: + reply_message_id = int(str(reply_message_raw)) + except (TypeError, ValueError): + reply_message_id = 0 + if reply_message_id != inbound_message_id: + return None, self._blocked_route_receipt( + "interaction_reply_context_missing_or_mismatched", + route_context={"signal_family": interaction_kind}, + ) + if actual_bot_alias in {"openclaw_bot", "nemotron_bot"} and ( + target_chat_id != str(settings.SRE_GROUP_CHAT_ID or "") + ): + return None, self._blocked_route_receipt( + "assistant_bot_interaction_destination_not_owned", + route_context={"signal_family": interaction_kind}, + ) + + return target_chat_id, { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "classifier": "non_monitoring_interaction_allowed", + "product_id": "awoooi", + "signal_family": interaction_kind, + "severity": "N/A", + "destination_alias": "inbound_interaction_reply_target", + "sender_bot_alias": actual_bot_alias, + "inbound_context_verified": True, + "provider_send_performed": False, + } + + def canonical_destination_chat_id( + self, + *, + product_id: str, + signal_family: str, + severity: str, + ) -> str: + """Resolve the runtime binding without exposing it in a receipt.""" + chat_id, _ = self._authorize_canonical_send_message( + payload={}, + route_context={ + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + }, + legacy_notification_type=None, + actual_bot_alias="tsenyang_bot", + ) + return chat_id or "" + def _summarize_callback_data_for_audit(self, callback_data: str) -> dict[str, str | None]: """Return a redaction-safe summary of callback_data without persisting nonce.""" parts = str(callback_data or "").split(":") @@ -5629,6 +5972,60 @@ class TelegramGateway: Returns: dict: API 回應 """ + payload = dict(payload) + route_context = payload.pop(_CANONICAL_ROUTE_CONTEXT_KEY, None) + legacy_notification_type = payload.pop( + _LEGACY_NOTIFICATION_TYPE_KEY, + None, + ) + interaction_context = payload.pop( + _NON_MONITORING_INTERACTION_KEY, + None, + ) + canonical_route_receipt: dict[str, object] | None = None + if method == "sendMessage": + if route_context is not None or legacy_notification_type is not None: + canonical_chat_id, canonical_route_receipt = ( + self._authorize_canonical_send_message( + payload=payload, + route_context=route_context, + legacy_notification_type=legacy_notification_type, + actual_bot_alias="tsenyang_bot", + ) + ) + elif interaction_context is not None: + canonical_chat_id, canonical_route_receipt = ( + self._authorize_non_monitoring_interaction( + payload=payload, + interaction_context=interaction_context, + actual_bot_alias="tsenyang_bot", + ) + ) + else: + canonical_chat_id, canonical_route_receipt = ( + None, + self._blocked_route_receipt( + "missing_product_signal_severity_or_interaction_context" + ), + ) + if canonical_chat_id is None: + logger.warning( + "telegram_canonical_route_blocked_no_egress", + classifier=canonical_route_receipt["classifier"], + product_id=canonical_route_receipt["product_id"], + signal_family=canonical_route_receipt["signal_family"], + severity=canonical_route_receipt["severity"], + provider_send_performed=False, + ) + return { + "ok": False, + "_awooop_outbound_mirror_acknowledged": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": canonical_route_receipt, + } + payload["chat_id"] = canonical_chat_id + if not self._initialized: await self.initialize() @@ -5637,6 +6034,11 @@ class TelegramGateway: payload = normalize_telegram_send_message_payload(method, payload) source_envelope_extra = payload.pop(_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY, None) + if canonical_route_receipt is not None: + source_envelope_extra = dict(source_envelope_extra or {}) + source_envelope_extra["canonical_route_receipt"] = ( + canonical_route_receipt + ) await self._attach_incident_thread_reply(method, payload) callback_reply = ( source_envelope_extra.get("callback_reply") @@ -5764,6 +6166,15 @@ class TelegramGateway: ) result["_awooop_provider_send_performed"] = True else: + if canonical_route_receipt is not None: + canonical_route_receipt["provider_send_performed"] = True + source_envelope_extra = ( + _acknowledge_callback_reply_source_envelope( + source_envelope_extra, + delivery_result=result, + provider_message_id=provider_message_id, + ) + ) await self._mirror_outbound_message( method=method, payload=payload, @@ -5775,6 +6186,12 @@ class TelegramGateway: result["_awooop_delivery_status"] = "pending_unknown" result["_awooop_provider_send_performed"] = True + if canonical_route_receipt is not None: + canonical_route_receipt["provider_send_performed"] = True + result["_awoooi_canonical_route_receipt"] = ( + canonical_route_receipt + ) + span.set_status(trace.Status(trace.StatusCode.OK)) return result @@ -5924,6 +6341,7 @@ class TelegramGateway: chunk_index: int, chunk_count: int, error: Exception, + parent_message_id: int | None = None, ) -> None: """Persist callback reply delivery failures into AwoooP outbound evidence.""" if not incident_id: @@ -5964,6 +6382,7 @@ class TelegramGateway: chunk_count=chunk_count, callback_action=callback_action, error=safe_error, + parent_message_id=parent_message_id, ), ) @@ -6358,11 +6777,30 @@ class TelegramGateway: "text": text, "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P2", + }, }) + if not _telegram_send_delivery_succeeded(result): + logger.info( + "analyzing_placeholder_no_send", + resource=resource_name, + delivery_status=( + result.get("_awooop_delivery_status") + if isinstance(result, Mapping) + else "invalid_response" + ), + ) + return None + msg_id: int | None = None result_val = result.get("result") if isinstance(result_val, dict): - msg_id = result_val.get("message_id") + candidate_message_id = result_val.get("message_id") + if isinstance(candidate_message_id, int): + msg_id = candidate_message_id logger.info("analyzing_placeholder_sent", message_id=msg_id, resource=resource_name) return msg_id except Exception as e: @@ -6579,17 +7017,15 @@ class TelegramGateway: controlled_auto_authorized=controlled_auto_authorized, ) - # 發送訊息：2026-04-30 統帥指示，告警卡片完整切到 SRE 戰情室群組。 - target_chat_id = self.alert_chat_id - if not target_chat_id: - logger.warning("telegram_approval_card_skipped", reason="alert_chat_id_missing") - return {} + # Unknown/missing notification_type is intentionally no-egress. The + # central sender resolves only allowlisted legacy types via the + # canonical registry and binds the actual runtime destination there. payload = { - "chat_id": target_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, "disable_web_page_preview": True, # 避免 SignOz URL 預覽 + _LEGACY_NOTIFICATION_TYPE_KEY: notification_type or "UNKNOWN", } logger.info( @@ -6597,26 +7033,50 @@ class TelegramGateway: approval_id=approval_id, risk_level=risk_level, resource=resource_name, - target_chat_id=str(target_chat_id), + destination_alias="canonical_policy_pending", signoz_integrated=signoz_metrics is not None, auto_tuning_available=bool(auto_tuning_command), ) result = await self._send_request("sendMessage", payload) - _msg_id = result.get("result", {}).get("message_id") - logger.info( - "telegram_approval_card_sent", - approval_id=approval_id, - message_id=_msg_id, - target_chat_id=str(target_chat_id), + delivery_succeeded = _telegram_send_delivery_succeeded(result) + result_value = result.get("result") + _msg_id = ( + result_value.get("message_id") + if delivery_succeeded and isinstance(result_value, dict) + else None ) + route_receipt = result.get("_awoooi_canonical_route_receipt") or {} + destination_alias = str(route_receipt.get("destination_alias") or "none") + runtime_target_chat_id = "" + if _msg_id and route_receipt.get("decision") == "allowed": + runtime_target_chat_id = self.canonical_destination_chat_id( + product_id=str(route_receipt.get("product_id") or ""), + signal_family=str(route_receipt.get("signal_family") or ""), + severity=str(route_receipt.get("severity") or ""), + ) + if delivery_succeeded: + logger.info( + "telegram_approval_card_sent", + approval_id=approval_id, + message_id=_msg_id, + destination_alias=destination_alias, + ) + else: + logger.warning( + "telegram_approval_card_no_send", + approval_id=approval_id, + destination_alias=destination_alias, + delivery_status=result.get("_awooop_delivery_status"), + ) # 2026-04-18 ADR-090-D: 寫入 notification_outcomes (MASTER §7.1 #10 KPI) try: from sqlalchemy import text as _sql + from src.db.base import get_db_context - _delivered = "delivered" if _msg_id else "failed" + _delivered = "delivered" if delivery_succeeded else "failed" _notif_type = f"TYPE-3-{alert_category}" if alert_category else "TYPE-3" async with get_db_context() as _db: await _db.execute( @@ -6632,7 +7092,7 @@ class TelegramGateway: { "aid": approval_id, "nt": _notif_type, - "rp": str(target_chat_id), + "rp": destination_alias, "mid": str(_msg_id) if _msg_id else None, "ds": _delivered, "md": '{"risk_level":"' + str(risk_level) + '"}', @@ -6643,7 +7103,7 @@ class TelegramGateway: # 2026-04-19 ogt + Claude Opus 4.7: 修 AP-1 — message_id 同時存進 # approval_records.telegram_message_id,不只 Redis(重啟會丟) - if _msg_id: + if _msg_id and runtime_target_chat_id: try: from src.services.approval_db import get_approval_service _svc = get_approval_service() @@ -6651,6 +7111,7 @@ class TelegramGateway: # 若有 update_telegram_message 方法(通常用 incident_id) # 先用 incident_id 更新,再 fallback 直接 UPDATE approval_records from sqlalchemy import text as _sql2 + from src.db.base import get_db_context as _gdc async with _gdc() as _db2: await _db2.execute( @@ -6662,7 +7123,7 @@ class TelegramGateway: """), { "mid": int(_msg_id), - "cid": int(target_chat_id), + "cid": int(runtime_target_chat_id), "aid": str(approval_id), }, ) @@ -6781,11 +7242,15 @@ class TelegramGateway: notification_type=notification_type, controlled_auto_authorized=controlled_auto_authorized, ) - resp = await self.send_to_group(text=text, reply_markup=_group_keyboard) + resp = await self.send_to_group( + text=text, + reply_markup=_group_keyboard, + legacy_notification_type=notification_type or "UNKNOWN", + ) # 2026-04-10 Claude Sonnet 4.6: 儲存 message_id 到 Redis，供 append_incident_update 使用 # tg_msg:{incident_id} → Telegram message_id (TTL 24h) - if incident_id and resp: + if incident_id and _telegram_send_delivery_succeeded(resp): tg_message_id = (resp.get("result") or {}).get("message_id") or resp.get("message_id") if tg_message_id: from src.core.redis_client import get_redis @@ -6846,10 +7311,10 @@ class TelegramGateway: return await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-1", }, ) @@ -6921,10 +7386,10 @@ class TelegramGateway: _result = await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-4D", }, ) @@ -6953,6 +7418,9 @@ class TelegramGateway: user_id: int, username: str, user: dict, + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> dict: """ 處理 drift_view / drift_adopt / drift_revert 按鈕。 @@ -6967,7 +7435,11 @@ class TelegramGateway: try: if action == "drift_view": await self._answer_callback(callback_query_id, action, text="🔍 撈全部 Diff...") - await self._send_drift_diff_detail(report_id) + await self._send_drift_diff_detail( + report_id, + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) return { "action": action, "approval_id": approval_id, "user": user, "success": True, "info_action": True, @@ -6984,6 +7456,8 @@ class TelegramGateway: _ok = False await self._edit_drift_card_outcome( report_id=report_id, verb="已採納", by=username, ok=_ok, + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, ) return {"action": action, "approval_id": approval_id, "user": user, "success": _ok} @@ -6998,6 +7472,8 @@ class TelegramGateway: _ok = False await self._edit_drift_card_outcome( report_id=report_id, verb="已回滾", by=username, ok=_ok, + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, ) return {"action": action, "approval_id": approval_id, "user": user, "success": _ok} @@ -7031,7 +7507,14 @@ class TelegramGateway: return "human_high" return "routine_medium" - async def _send_drift_diff_detail(self, report_id: str, page: int = 0) -> None: + async def _send_drift_diff_detail( + self, + report_id: str, + page: int = 0, + *, + inbound_chat_id: str | int, + inbound_message_id: int, + ) -> None: """ 送分頁 Drift Diff 到 Telegram (drift_view / drift_view_page 按鈕回應) @@ -7043,9 +7526,15 @@ class TelegramGateway: _rpt = await get_drift_repository().get_by_id(report_id) if not _rpt: await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": f"⚠️ 找不到 Drift report {html.escape(report_id)}", "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, }) return @@ -7125,10 +7614,16 @@ class TelegramGateway: _keyboard = {"inline_keyboard": _rows} if _rows else None _payload = { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": _full, "parse_mode": "HTML", "disable_web_page_preview": True, + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, } if _keyboard: _payload["reply_markup"] = _keyboard @@ -7136,9 +7631,15 @@ class TelegramGateway: except Exception as _e: logger.warning("drift_diff_detail_send_failed", report_id=report_id, page=page, error=str(_e)) await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": f"⚠️ Drift Diff 查詢失敗: {html.escape(str(_e)[:150])}", "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, }) async def _handle_ai_advisory_action( @@ -7149,7 +7650,9 @@ class TelegramGateway: user_id: int, username: str, user: dict, - message_id: int | None = None, + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> dict: """ 2026-04-19 P0 修 (ADR-092): 處理 4 LLM scanner 的互動按鈕. @@ -7191,14 +7694,27 @@ class TelegramGateway: await self._answer_callback(callback_query_id, action, text=feedback_text) # 2026-04-22 Claude Sonnet 4.6: 發群組 reply（toast 2-3 秒消失，群組才是永久可見） - if message_id and feedback_text: + if feedback_text: try: - await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + group_reply_result = await self._send_request("sendMessage", { + "chat_id": inbound_chat_id, "text": feedback_text, - "reply_to_message_id": message_id, + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, }) - logger.info("ai_advisory_group_reply_sent", action=pure_action, message_id=message_id) + if not _telegram_send_delivery_succeeded(group_reply_result): + raise TelegramGatewayError( + "ai_advisory_group_reply_delivery_not_proven" + ) + logger.info( + "ai_advisory_group_reply_sent", + action=pure_action, + message_id=inbound_message_id, + ) except Exception as _ge: logger.warning("ai_advisory_group_reply_failed", action=pure_action, error=str(_ge)) @@ -7217,6 +7733,9 @@ class TelegramGateway: async def _edit_drift_card_outcome( self, report_id: str, verb: str, by: str, ok: bool, + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> None: """ drift_adopt/drift_revert 執行後: @@ -7229,29 +7748,52 @@ class TelegramGateway: f"({'成功' if ok else '失敗'})\n" f"Drift {html.escape(report_id)}" ) - _msg_id: int | None = None + # Redis is historical evidence only. A stale tg_drift value must never + # replace the verified message identity carried by this callback. try: _msg_id_raw = await get_redis().get(f"tg_drift:{report_id}") if _msg_id_raw: - _msg_id = int(_msg_id_raw) - # 先移除按鈕 - await self._send_request("editMessageReplyMarkup", { - "chat_id": self.alert_chat_id, - "message_id": _msg_id, - "reply_markup": {"inline_keyboard": []}, - }) + stored_message_id = int(_msg_id_raw) + if stored_message_id != inbound_message_id: + logger.warning( + "drift_parent_receipt_stale_ignored", + report_id=report_id, + stored_message_id=stored_message_id, + verified_inbound_message_id=inbound_message_id, + ) except Exception as _e: - logger.warning("drift_card_buttons_remove_failed", report_id=report_id, error=str(_e)) + logger.warning( + "drift_parent_receipt_read_failed_ignored", + report_id=report_id, + error=str(_e), + ) + + try: + await self._send_request("editMessageReplyMarkup", { + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, + "reply_markup": {"inline_keyboard": []}, + }) + except Exception as _e: + logger.warning( + "drift_card_buttons_remove_failed", + report_id=report_id, + error=str(_e), + ) # 送簽核戳訊息（reply_to 原卡片,若有 msg_id） try: _payload: dict = { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": _stamp, "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, } - if _msg_id: - _payload["reply_to_message_id"] = _msg_id await self._send_request("sendMessage", _payload) except Exception as _e: logger.warning("drift_outcome_stamp_send_failed", report_id=report_id, error=str(_e)) @@ -7318,10 +7860,10 @@ class TelegramGateway: return await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-8M", _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: { "source_refs": source_refs, }, @@ -7383,10 +7925,10 @@ class TelegramGateway: return await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-5S", }, ) @@ -7425,14 +7967,14 @@ class TelegramGateway: # 鐵律: 寧可沒按鈕，不可有死按鈕 (feedback_no_ghost_buttons.md) keyboard = {"inline_keyboard": []} - target_chat = group_chat_id or self.alert_chat_id return await self._send_request( "sendMessage", { - "chat_id": target_chat, + "chat_id": group_chat_id or "", "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-6B", }, ) @@ -7477,14 +8019,14 @@ class TelegramGateway: # 鐵律: 寧可沒按鈕，不可有死按鈕 (feedback_no_ghost_buttons.md) keyboard = {"inline_keyboard": []} - target_chat = group_chat_id or self.alert_chat_id return await self._send_request( "sendMessage", { - "chat_id": target_chat, - "text": text + ("\n📣 @所有人 事故升級，請協助！" if settings.SRE_GROUP_CHAT_ID else ""), + "chat_id": group_chat_id or "", + "text": text + "\n📣 @所有人 事故升級，請協助！", "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-7E", }, ) @@ -7572,11 +8114,27 @@ class TelegramGateway: "parse_mode": "HTML", "reply_markup": self._build_sentry_keyboard(error_id), "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P2", + }, } logger.info("telegram_sentry_error_sending", error_id=error_id, service=service_name) result = await self._send_request("sendMessage", payload) - logger.info("telegram_sentry_error_sent", error_id=error_id) + if _telegram_send_delivery_succeeded(result): + logger.info( + "telegram_sentry_error_routed", + error_id=error_id, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) + else: + logger.warning( + "telegram_sentry_error_no_send", + error_id=error_id, + delivery_status=result.get("_awooop_delivery_status"), + ) return result @@ -7630,11 +8188,20 @@ class TelegramGateway: "parse_mode": "HTML", "reply_markup": self._build_resource_keyboard(resource_id), "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P2", + }, } logger.info("telegram_resource_warning_sending", resource_id=resource_id, pod=pod_name) result = await self._send_request("sendMessage", payload) - logger.info("telegram_resource_warning_sent", resource_id=resource_id) + logger.info( + "telegram_resource_warning_routed", + resource_id=resource_id, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7686,11 +8253,20 @@ class TelegramGateway: "text": message.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P3", + }, } logger.info("telegram_repair_report_sending", date=report_date) result = await self._send_request("sendMessage", payload) - logger.info("telegram_repair_report_sent", date=report_date) + logger.info( + "telegram_repair_report_routed", + date=report_date, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7743,11 +8319,20 @@ class TelegramGateway: "text": message.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P3", + }, } logger.info("telegram_daily_summary_sending", date=summary_date) result = await self._send_request("sendMessage", payload) - logger.info("telegram_daily_summary_sent", date=summary_date) + logger.info( + "telegram_daily_summary_routed", + date=summary_date, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7801,6 +8386,11 @@ class TelegramGateway: "text": msg.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P2", + }, } logger.info("telegram_cicd_progress_sending", job=job_name, status=status) @@ -7812,7 +8402,13 @@ class TelegramGateway: result = await self._send_request("sendMessage", payload) span.set_attribute("telegram.attempts", attempt + 1) span.set_status(trace.Status(trace.StatusCode.OK)) - logger.info("telegram_cicd_progress_sent", job=job_name, status=status, attempt=attempt + 1) + logger.info( + "telegram_cicd_progress_routed", + job=job_name, + status=status, + attempt=attempt + 1, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result except TelegramGatewayError as e: last_error = e @@ -7881,11 +8477,20 @@ class TelegramGateway: "text": message.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P3", + }, } logger.info("telegram_deploy_success_sending", commit=commit_sha[:8]) result = await self._send_request("sendMessage", payload) - logger.info("telegram_deploy_success_sent", commit=commit_sha[:8]) + logger.info( + "telegram_deploy_success_routed", + commit=commit_sha[:8], + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7924,11 +8529,20 @@ class TelegramGateway: "text": message.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "business_finops", + "severity": "P2", + }, } logger.info("telegram_rate_limit_warning_sending", provider=provider) result = await self._send_request("sendMessage", payload) - logger.info("telegram_rate_limit_warning_sent", provider=provider) + logger.info( + "telegram_rate_limit_warning_routed", + provider=provider, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7940,6 +8554,7 @@ class TelegramGateway: message_id: int, original_text: str = "", username: str = "", + chat_id: str | int | None = None, ) -> dict: """ 處理簽核/調優回調 @@ -7956,6 +8571,24 @@ class TelegramGateway: dict: 處理結果 {action, approval_id, user, auto_tuning_result?} """ try: + verified_inbound_chat_id = str(chat_id or "") + try: + verified_inbound_message_id = int(message_id) + except (TypeError, ValueError): + verified_inbound_message_id = 0 + if not verified_inbound_chat_id or verified_inbound_message_id <= 0: + logger.warning( + "telegram_callback_blocked_missing_inbound_context", + callback_data_prefix=callback_data.split(":", 1)[0], + ) + return { + "action": "callback_blocked", + "approval_id": "", + "user": {"id": user_id, "username": username}, + "success": False, + "reason": "missing_verified_interaction_context", + } + # =================================================================== # Step 0: LLM Action Callback（H1/B4）— la:{short_id} 格式優先路由 # 2026-04-27 Claude Sonnet 4.6: H1+B4 Fix — 鬼魂按鈕鐵律修復 @@ -7967,6 +8600,8 @@ class TelegramGateway: callback_data=callback_data, user_id=user_id, username=username, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) # =================================================================== @@ -7990,17 +8625,29 @@ class TelegramGateway: # ADR-050 P2: 取得事件詳情，傳送新訊息 (保留原始簽核卡片+按鈕) # 2026-04-01 Claude Code (ADR-050 P2) await self._answer_callback_nonfatal(callback_query_id, action, text="📋 詳情傳送中...") - await self._send_incident_detail(incident_id) + await self._send_incident_detail( + incident_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, + ) elif action == "history": # ADR-050 P2: 取得頻率統計 # 2026-04-01 Claude Code (ADR-050 P2) await self._answer_callback_nonfatal(callback_query_id, action, text="📊 歷史統計傳送中...") - await self._send_incident_history(incident_id) + await self._send_incident_history( + incident_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, + ) elif action == "reanalyze": # ADR-050 P2: 觸發重診 # 2026-04-01 Claude Code (ADR-050 P2): reanalyze button handler await self._answer_callback_nonfatal(callback_query_id, action, text="🔄 重診排程中...") - await self._send_reanalyze_result(incident_id) + await self._send_reanalyze_result( + incident_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, + ) elif action == "drift_view_page": # 2026-04-20 P0.1 ogt + Claude Opus 4.7: drift_view 分頁切頁 # incident_id 格式: {report_id}_{page}（底線分隔） @@ -8012,7 +8659,12 @@ class TelegramGateway: await self._answer_callback_nonfatal( callback_query_id, action, text=f"📄 切換至第 {_page_num + 1} 頁..." ) - await self._send_drift_diff_detail(_rid or incident_id, page=_page_num) + await self._send_drift_diff_detail( + _rid or incident_id, + page=_page_num, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, + ) else: # 2026-04-14 Claude Sonnet 4.6 (Phase 5 Sprint 5.1): # 未知 action → fallback dispatcher (查看 callback_action_spec.yaml 是否有註冊) @@ -8021,6 +8673,8 @@ class TelegramGateway: action=action, incident_id=incident_id, user_id=user_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) return { @@ -8053,7 +8707,8 @@ class TelegramGateway: guard_result = await self._check_incident_state_guard( approval_id=approval_id, callback_query_id=callback_query_id, - message_id=message_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, original_text=original_text, ) if guard_result is not None: @@ -8072,6 +8727,8 @@ class TelegramGateway: user_id=user_id, username=username, user=user, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) # =================================================================== @@ -8088,7 +8745,8 @@ class TelegramGateway: user_id=user_id, username=username, user=user, - message_id=message_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) # =================================================================== @@ -8179,22 +8837,20 @@ class TelegramGateway: labels=_labels, ) - # Reply 結果到原告警卡片 - try: - from src.core.redis_client import get_redis as _gr - _rds = _gr() - _msg_id_raw = await _rds.get(f"tg_msg:{_incident_id_resolved}") - _orig_msg = int(_msg_id_raw) if _msg_id_raw else None - except Exception: - _orig_msg = None + # Reply to the exact verified callback message. A Redis parent + # lookup cannot replace the inbound identity of this interaction. try: _payload = { - "chat_id": self.alert_chat_id, + "chat_id": verified_inbound_chat_id, "text": _result.result_text, "parse_mode": "HTML", + "reply_to_message_id": verified_inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": verified_inbound_chat_id, + "inbound_message_id": verified_inbound_message_id, + }, } - if _orig_msg: - _payload["reply_to_message_id"] = _orig_msg await self._send_request("sendMessage", _payload) except Exception as _re: logger.warning("category_action_reply_send_failed", error=str(_re)) @@ -8236,11 +8892,12 @@ class TelegramGateway: ) # 更新訊息 await self._update_message_after_action( - message_id=message_id, action="tune", username=username, original_text=original_text, extra_info=auto_tuning_result.get("command", ""), + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) return { @@ -8265,10 +8922,11 @@ class TelegramGateway: text="⏰ 30 分鐘後再提醒", ) await self._update_message_after_action( - message_id=message_id, action="snooze", username=username, original_text=original_text, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) return { "action": action, @@ -8290,11 +8948,12 @@ class TelegramGateway: text="🔕 此類告警靜默 1 小時", ) await self._update_message_after_action( - message_id=message_id, action="silence", username=username, original_text=original_text, extra_info=silence_result.get("resource_name", ""), + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) return { "action": action, @@ -8311,6 +8970,18 @@ class TelegramGateway: # 實際步驟收集在 handle_message() 的 /done 流程中完成 # =================================================================== if action == "log_manual_fix": + if not chat_id or not message_id: + logger.warning( + "manual_fix_prompt_blocked_missing_inbound_context", + approval_id=approval_id, + ) + return { + "action": action, + "approval_id": approval_id, + "user": user, + "success": False, + "reason": "missing_verified_interaction_context", + } await self._answer_callback( callback_query_id, "log_manual_fix", @@ -8330,7 +9001,7 @@ class TelegramGateway: await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": chat_id, "text": ( "📝 手動修復記錄\n" "━━━━━━━━━━━━━━━━━━━\n" @@ -8339,6 +9010,12 @@ class TelegramGateway: "30 分鐘內有效" ), "parse_mode": "HTML", + "reply_to_message_id": message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(chat_id), + "inbound_message_id": message_id, + }, }, ) return { @@ -8358,10 +9035,11 @@ class TelegramGateway: # Step 4: 更新訊息 (保留原始內容 + 簽核鋼印) # =================================================================== await self._update_message_after_action( - message_id=message_id, action=action, username=username, original_text=original_text, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) logger.info( @@ -8409,7 +9087,9 @@ class TelegramGateway: self, approval_id: str, callback_query_id: str, - message_id: int, + *, + inbound_chat_id: str | int, + inbound_message_id: int, original_text: str, ) -> dict | None: """ @@ -8453,8 +9133,8 @@ class TelegramGateway: safe_original = html.escape(original_text) if original_text else "" stamp = _format_resolved_guard_stamp(incident.resolved_at) await self._send_request("editMessageText", { - "chat_id": self.alert_chat_id, - "message_id": message_id, + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, "text": f"{safe_original}\n{separator}\n{stamp}" if safe_original else stamp, "parse_mode": "HTML", "reply_markup": {"inline_keyboard": []}, @@ -8464,8 +9144,8 @@ class TelegramGateway: # 移除按鈕保底 try: await self._send_request("editMessageReplyMarkup", { - "chat_id": self.alert_chat_id, - "message_id": message_id, + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, "reply_markup": {"inline_keyboard": []}, }) except Exception: @@ -8502,6 +9182,8 @@ class TelegramGateway: user_id: int, username: str, fix_steps: str, + chat_id: str | int | None = None, + message_id: int | None = None, ) -> dict: """ ADR-071-H: 處理使用者輸入 /done 後的手動修復步驟記錄 @@ -8519,6 +9201,25 @@ class TelegramGateway: username: Telegram username fix_steps: 使用者輸入的修復步驟 """ + if not chat_id or not message_id: + return { + "success": False, + "reason": "missing_verified_interaction_context", + } + + def manual_reply_payload(text: str) -> dict[str, object]: + return { + "chat_id": chat_id, + "text": text, + "parse_mode": "HTML", + "reply_to_message_id": message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "operator_command_reply", + "inbound_chat_id": str(chat_id), + "inbound_message_id": message_id, + }, + } + try: from src.core.redis_client import get_redis as _get_redis redis = _get_redis() @@ -8526,11 +9227,10 @@ class TelegramGateway: pending_key = f"manual_fix_pending:{user_id}" approval_id_bytes = await redis.get(pending_key) if not approval_id_bytes: - await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, - "text": "⚠️ 找不到待記錄的修復任務，或已逾時。", - "parse_mode": "HTML", - }) + await self._send_request( + "sendMessage", + manual_reply_payload("⚠️ 找不到待記錄的修復任務，或已逾時。"), + ) return {"success": False, "reason": "no_pending_task"} approval_id = approval_id_bytes.decode() if isinstance(approval_id_bytes, bytes) else str(approval_id_bytes) @@ -8543,11 +9243,12 @@ class TelegramGateway: approval_repo = ApprovalDBRepository() approval = await approval_repo.get_by_approval_id(approval_id) if not approval: - await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, - "text": f"⚠️ 找不到簽核單 {html.escape(approval_id)}", - "parse_mode": "HTML", - }) + await self._send_request( + "sendMessage", + manual_reply_payload( + f"⚠️ 找不到簽核單 {html.escape(approval_id)}" + ), + ) return {"success": False, "reason": "approval_not_found"} incident_repo = IncidentDBRepository() @@ -8599,17 +9300,16 @@ class TelegramGateway: ) # 回覆確認 - await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, - "text": ( + await self._send_request( + "sendMessage", + manual_reply_payload( f"✅ 手動修復步驟已記錄\n" f"━━━━━━━━━━━━━━━━━━━\n" f"📋 事件: {html.escape(approval.incident_id)}\n" f"👤 記錄者: @{html.escape(username or str(user_id))}\n\n" f"正在建立草稿 Playbook，請至 AWOOOI 審核後生效。" ), - "parse_mode": "HTML", - }) + ) logger.info( "manual_fix_recorded", @@ -8808,6 +9508,9 @@ class TelegramGateway: callback_data: str, user_id: int, username: str = "", + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> dict: """ B4: 處理 LLM 動態按鈕 callback（格式 la:{short_id}） @@ -8945,9 +9648,15 @@ class TelegramGateway: ) try: await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": result_text, "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, }) except Exception as exc: logger.warning("llm_action_result_notify_failed", error=str(exc)) @@ -9007,10 +9716,12 @@ class TelegramGateway: async def _update_message_after_action( self, - message_id: int, action: str, username: str, original_text: str, + *, + inbound_chat_id: str | int, + inbound_message_id: int, extra_info: str = "", ) -> None: """ @@ -9043,12 +9754,16 @@ class TelegramGateway: # 先用 editMessageReplyMarkup 確保按鈕移除，再嘗試更新文字 try: await self._send_request("editMessageReplyMarkup", { - "chat_id": self.alert_chat_id, - "message_id": message_id, + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, "reply_markup": {"inline_keyboard": []}, }) except TelegramGatewayError as e: - logger.warning("telegram_remove_buttons_failed", message_id=message_id, error=str(e)) + logger.warning( + "telegram_remove_buttons_failed", + message_id=inbound_message_id, + error=str(e), + ) # Step 2: 嘗試更新文字 (原始文字已轉義，確保 HTML 安全) separator = "──────────────" @@ -9056,8 +9771,8 @@ class TelegramGateway: safe_updated_text = f"{safe_original}\n{separator}\n{stamp}" try: await self._send_request("editMessageText", { - "chat_id": self.alert_chat_id, - "message_id": message_id, + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, "text": safe_updated_text, "parse_mode": "HTML", "reply_markup": {"inline_keyboard": []}, @@ -9065,7 +9780,11 @@ class TelegramGateway: }) except TelegramGatewayError as e: # 文字更新失敗不影響整體流程，按鈕已移除 - logger.warning("telegram_update_text_failed", message_id=message_id, error=str(e)) + logger.warning( + "telegram_update_text_failed", + message_id=inbound_message_id, + error=str(e), + ) async def mark_auto_repaired( self, @@ -9101,7 +9820,7 @@ class TelegramGateway: # 回覆原訊息說明結果 _status = "✅ 已自動修復" if success else "❌ 自動修復失敗" - await self._send_request("sendMessage", { + result = await self._send_request("sendMessage", { "chat_id": self.alert_chat_id, "text": ( f"{_status}\n" @@ -9110,7 +9829,27 @@ class TelegramGateway: ), "parse_mode": "HTML", "reply_parameters": {"message_id": message_id}, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: ( + _incident_lifecycle_parent_source_envelope_extra( + approval_id=approval_id, + parent_lookup_key=f"tg_approval:{approval_id}", + parent_message_id=message_id, + event_kind="auto_repair_result", + ) + ), }) + if not _telegram_send_delivery_succeeded(result): + logger.warning( + "mark_auto_repaired_no_send", + approval_id=approval_id, + delivery_status=result.get("_awooop_delivery_status"), + ) + return False return True except Exception as e: @@ -9163,14 +9902,17 @@ class TelegramGateway: # 詳細執行紀錄應進 timeline / AwoooP Run Monitor，避免群組被 auto-failure 洗版。 status_hash = hashlib.sha1(status_line.encode("utf-8")).hexdigest()[:16] dedup_key = f"{INCIDENT_UPDATE_DEDUP_PREFIX}{incident_id}:{status_hash}" - try: - was_set = await redis.set( - dedup_key, - "1", - ex=INCIDENT_UPDATE_DEDUP_TTL_SECONDS, - nx=True, + if dedup_key in self._delivered_incident_update_dedup: + logger.info( + "append_incident_update_dedup_suppressed", + incident_id=incident_id, + dedup_key=dedup_key, ) - if not was_set: + return True + try: + durable_dedup = await redis.get(dedup_key) + if durable_dedup in {"provider_ack", b"provider_ack"}: + self._delivered_incident_update_dedup.add(dedup_key) logger.info( "append_incident_update_dedup_suppressed", incident_id=incident_id, @@ -9179,36 +9921,37 @@ class TelegramGateway: return True except Exception as exc: logger.warning( - "append_incident_update_dedup_failed", + "append_incident_update_dedup_read_failed", incident_id=incident_id, error=str(exc), ) suppress_reply = False + global_dedup_key: str | None = None if _is_noisy_failure_update(status_line): # 不同 incident 若卡在同一個自動修復/診斷失敗摘要，Telegram 只推第一則； # 每個 incident 仍會繼續移除原卡危險按鈕，完整細節交給 timeline / AwoooP。 global_hash = hashlib.sha1(status_line.encode("utf-8")).hexdigest()[:16] global_dedup_key = f"{INCIDENT_UPDATE_GLOBAL_FAILURE_DEDUP_PREFIX}{global_hash}" - try: - was_global_set = await redis.set( - global_dedup_key, - incident_id, - ex=INCIDENT_UPDATE_GLOBAL_FAILURE_DEDUP_TTL_SECONDS, - nx=True, - ) - suppress_reply = not bool(was_global_set) - if suppress_reply: - logger.info( - "append_incident_update_global_failure_dedup_suppressed", + if global_dedup_key in self._delivered_global_failure_dedup: + suppress_reply = True + else: + try: + durable_global_dedup = await redis.get(global_dedup_key) + if durable_global_dedup in {"provider_ack", b"provider_ack"}: + self._delivered_global_failure_dedup.add(global_dedup_key) + suppress_reply = True + except Exception as exc: + logger.warning( + "append_incident_update_global_failure_dedup_read_failed", incident_id=incident_id, - dedup_key=global_dedup_key, + error=str(exc), ) - except Exception as exc: - logger.warning( - "append_incident_update_global_failure_dedup_failed", + if suppress_reply: + logger.info( + "append_incident_update_global_failure_dedup_suppressed", incident_id=incident_id, - error=str(exc), + dedup_key=global_dedup_key, ) # Step 1: 取得原始訊息文字（Telegram Bot API 不提供讀取原文，只能在 editMessageText 裡重建） @@ -9246,15 +9989,69 @@ class TelegramGateway: # Step 2: Reply 原訊息追加狀態（保留原文不動，以 reply 方式延續） try: - await self._send_request("sendMessage", { + result = await self._send_request("sendMessage", { "chat_id": self.alert_chat_id, "text": status_line, "parse_mode": "HTML", "reply_to_message_id": message_id, "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: ( + _incident_lifecycle_parent_source_envelope_extra( + incident_id=incident_id, + parent_lookup_key=f"tg_msg:{incident_id}", + parent_message_id=message_id, + event_kind="incident_status_update", + ) + ), }) except TelegramGatewayError as e: logger.warning("append_incident_update_reply_failed", message_id=message_id, error=str(e)) + return False + + if not _telegram_send_delivery_succeeded(result): + logger.warning( + "append_incident_update_no_send", + incident_id=incident_id, + message_id=message_id, + delivery_status=result.get("_awooop_delivery_status"), + ) + return False + + self._delivered_incident_update_dedup.add(dedup_key) + try: + await redis.set( + dedup_key, + "provider_ack", + ex=INCIDENT_UPDATE_DEDUP_TTL_SECONDS, + nx=True, + ) + except Exception as exc: + logger.warning( + "append_incident_update_dedup_write_failed", + incident_id=incident_id, + error=str(exc), + ) + + if global_dedup_key: + self._delivered_global_failure_dedup.add(global_dedup_key) + try: + await redis.set( + global_dedup_key, + "provider_ack", + ex=INCIDENT_UPDATE_GLOBAL_FAILURE_DEDUP_TTL_SECONDS, + nx=True, + ) + except Exception as exc: + logger.warning( + "append_incident_update_global_failure_dedup_write_failed", + incident_id=incident_id, + error=str(exc), + ) logger.info( "append_incident_update_done", @@ -9297,14 +10094,17 @@ class TelegramGateway: return False dedup_key = f"{GROUPED_ALERT_DIGEST_DEDUP_PREFIX}{group_key}" - try: - was_set = await redis.set( - dedup_key, - incident_id, - ex=GROUPED_ALERT_DIGEST_DEDUP_TTL_SECONDS, - nx=True, + if dedup_key in self._delivered_grouped_digest_dedup: + logger.info( + "grouped_alert_digest_dedup_suppressed", + incident_id=incident_id, + group_key=group_key, ) - if not was_set: + return True + try: + durable_dedup = await redis.get(dedup_key) + if durable_dedup in {"provider_ack", b"provider_ack"}: + self._delivered_grouped_digest_dedup.add(dedup_key) logger.info( "grouped_alert_digest_dedup_suppressed", incident_id=incident_id, @@ -9313,14 +10113,14 @@ class TelegramGateway: return True except Exception as exc: logger.warning( - "grouped_alert_digest_dedup_failed", + "grouped_alert_digest_dedup_read_failed", incident_id=incident_id, group_key=group_key, error=str(exc), ) try: - await self._send_request("sendMessage", { + result = await self._send_request("sendMessage", { "chat_id": self.alert_chat_id, "text": digest_text[:1400], "parse_mode": "HTML", @@ -9329,6 +10129,19 @@ class TelegramGateway: "allow_sending_without_reply": True, }, "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: ( + _incident_lifecycle_parent_source_envelope_extra( + incident_id=incident_id, + parent_lookup_key=f"tg_msg:{incident_id}", + parent_message_id=message_id, + event_kind="grouped_alert_digest", + ) + ), }) except TelegramGatewayError as exc: logger.warning( @@ -9340,6 +10153,32 @@ class TelegramGateway: ) return False + if not _telegram_send_delivery_succeeded(result): + logger.warning( + "grouped_alert_digest_no_send", + incident_id=incident_id, + group_key=group_key, + message_id=message_id, + delivery_status=result.get("_awooop_delivery_status"), + ) + return False + + self._delivered_grouped_digest_dedup.add(dedup_key) + try: + await redis.set( + dedup_key, + "provider_ack", + ex=GROUPED_ALERT_DIGEST_DEDUP_TTL_SECONDS, + nx=True, + ) + except Exception as exc: + logger.warning( + "grouped_alert_digest_dedup_write_failed", + incident_id=incident_id, + group_key=group_key, + error=str(exc), + ) + logger.info( "grouped_alert_digest_reply_sent", incident_id=incident_id, @@ -9354,6 +10193,9 @@ class TelegramGateway: action: str, incident_id: str, user_id: int, + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> None: """ Phase 5 Sprint 5.1 (2026-04-14 Claude Sonnet 4.6): @@ -9397,24 +10239,23 @@ class TelegramGateway: labels=labels, ) - # Reply to 原卡片 — 從 Redis tg_msg 查 message_id - try: - from src.core.redis_client import get_redis - redis = get_redis() - msg_id_raw = await redis.get(f"tg_msg:{incident_id}") - orig_msg_id = int(msg_id_raw) if msg_id_raw else None - except Exception: - orig_msg_id = None - try: payload: dict = { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": result.result_text, "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, } - if orig_msg_id: - payload["reply_to_message_id"] = orig_msg_id - await self._send_request("sendMessage", payload) + reply_result = await self._send_request("sendMessage", payload) + if not _telegram_send_delivery_succeeded(reply_result): + raise TelegramGatewayError( + "category_action_reply_delivery_not_proven" + ) logger.info( "category_action_reply_sent", action=action, @@ -9425,7 +10266,13 @@ class TelegramGateway: except Exception as _e: logger.warning("category_action_reply_failed", action=action, error=str(_e)) - async def _send_incident_detail(self, incident_id: str) -> None: + async def _send_incident_detail( + self, + incident_id: str, + *, + inbound_chat_id: str | int, + inbound_message_id: int, + ) -> None: """ ADR-050 P2: 傳送事件詳情訊息 (不修改原始簽核卡片) @@ -9440,7 +10287,13 @@ class TelegramGateway: incident = await repo.get_by_id(incident_id) if not incident: - await self.send_notification(f"⚠️ 找不到事件 {html.escape(incident_id)}") + await self.send_notification( + f"⚠️ 找不到事件 {html.escape(incident_id)}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) return dc = incident.decision_chain @@ -9625,6 +10478,8 @@ class TelegramGateway: await self._send_html_line_message( lines, failure_context="incident_detail", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, reply_markup=_awooop_truth_chain_reply_markup(incident_id), incident_id=incident_id, callback_action="detail", @@ -9634,9 +10489,21 @@ class TelegramGateway: except Exception as e: logger.warning("send_incident_detail_failed", incident_id=incident_id, error=str(e)) - await self.send_notification(f"⚠️ 無法取得事件詳情: {html.escape(str(e)[:100])}") + await self.send_notification( + f"⚠️ 無法取得事件詳情: {html.escape(str(e)[:100])}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) - async def _send_incident_history(self, incident_id: str) -> None: + async def _send_incident_history( + self, + incident_id: str, + *, + inbound_chat_id: str | int, + inbound_message_id: int, + ) -> None: """ ADR-050 P2: 傳送事件頻率統計訊息與 DB truth-chain 摘要 @@ -9653,7 +10520,13 @@ class TelegramGateway: incident = await repo.get_by_id(incident_id) if not incident: - await self.send_notification(f"⚠️ 找不到事件 {html.escape(incident_id)}") + await self.send_notification( + f"⚠️ 找不到事件 {html.escape(incident_id)}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) return lines = [ @@ -9810,6 +10683,8 @@ class TelegramGateway: await self._send_html_line_message( lines, failure_context="incident_history", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, reply_markup=_awooop_truth_chain_reply_markup(incident_id), incident_id=incident_id, callback_action="history", @@ -9819,9 +10694,21 @@ class TelegramGateway: except Exception as e: logger.warning("send_incident_history_failed", incident_id=incident_id, error=str(e)) - await self.send_notification(f"⚠️ 無法取得歷史統計: {html.escape(str(e))}") + await self.send_notification( + f"⚠️ 無法取得歷史統計: {html.escape(str(e))}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) - async def _send_reanalyze_result(self, incident_id: str) -> None: + async def _send_reanalyze_result( + self, + incident_id: str, + *, + inbound_chat_id: str | int, + inbound_message_id: int, + ) -> None: """ ADR-050 P2: 觸發重診並傳送結果訊息 @@ -9856,12 +10743,22 @@ class TelegramGateway: f"{html.escape(result['message'])}" ) - await self.send_notification(msg) + await self.send_notification( + msg, + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) except Exception as e: logger.warning("send_reanalyze_result_failed", incident_id=incident_id, error=str(e)) await self.send_notification( - f"⚠️ 重診觸發失敗: {html.escape(str(e)[:100])}" + f"⚠️ 重診觸發失敗: {html.escape(str(e)[:100])}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, ) # ========================================================================= @@ -9886,7 +10783,12 @@ class TelegramGateway: "━━━━━━━━━━━━━━━━━\n" "⚠️ 請人工評估並手動處理" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="security_incident", + severity="P1", + text=text, + ) except Exception as e: logger.error("t1_guardrail_blocked_notify_failed", service=service_name, error=str(e)) @@ -9913,7 +10815,12 @@ class TelegramGateway: "━━━━━━━━━━━━━━━━━\n" "請等待備份完成後，人工重新評估修復方案" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="disaster_recovery", + severity="P1", + text=text, + ) except Exception as e: logger.error("t2_preflight_failed_notify_failed", service=service_name, error=str(e)) @@ -9939,7 +10846,12 @@ class TelegramGateway: f"錯誤: {err}\n" "請人工介入，備份異常" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="disaster_recovery", + severity="P1", + text=text, + ) except Exception as e: logger.error("t3_backup_result_notify_failed", backup=backup_name, error=str(e)) @@ -9964,7 +10876,12 @@ class TelegramGateway: "━━━━━━━━━━━━━━━━━\n" "請第二位審核者登入確認" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + ) except Exception as e: logger.error("t4_multisig_waiting_notify_failed", approval=approval_id, error=str(e)) @@ -9981,7 +10898,12 @@ class TelegramGateway: f"服務: {html.escape(service_name)}\n" "授權: 2/2 票 開始執行..." ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + ) except Exception as e: logger.error("t5_multisig_approved_notify_failed", service=service_name, error=str(e)) @@ -10000,7 +10922,12 @@ class TelegramGateway: f"動作: {html.escape(action_description)}\n" f"時間: {html.escape(timestamp)}" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + ) except Exception as e: logger.error("t6_change_applied_notify_failed", operator=operator, error=str(e)) @@ -10009,16 +10936,15 @@ class TelegramGateway: text: str, parse_mode: str = "HTML", chat_id: str | int | None = None, + *, + interaction_kind: str | None = None, + inbound_chat_id: str | int | None = None, + inbound_message_id: int | None = None, ) -> dict: - """ - 發送純文字通知 + """Reply to a verified inbound interaction; never classify by default. - Args: - text: 訊息內容 - parse_mode: 解析模式 - - Returns: - dict: API 回應 + A generic call without explicit inbound chat/message identity reaches + the final-exit gate without any interaction classifier and is blocked. """ payload_text = text[:500] payload_parse_mode = parse_mode @@ -10027,9 +10953,17 @@ class TelegramGateway: payload_parse_mode = None payload = { - "chat_id": chat_id or self.alert_chat_id, + "chat_id": chat_id or "", "text": payload_text, # SOUL.md 字數限制 } + if interaction_kind: + payload[_NON_MONITORING_INTERACTION_KEY] = { + "interaction_kind": interaction_kind, + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": inbound_message_id, + } + if inbound_message_id is not None: + payload["reply_to_message_id"] = inbound_message_id if payload_parse_mode: payload["parse_mode"] = payload_parse_mode @@ -10038,9 +10972,17 @@ class TelegramGateway: except TelegramGatewayError as exc: if payload_parse_mode and payload_parse_mode.upper() == "HTML" and "HTTP error: 400" in str(exc): fallback_payload = { - "chat_id": chat_id or self.alert_chat_id, + "chat_id": chat_id or "", "text": _plain_text_from_html(text, limit=500), } + if interaction_kind: + fallback_payload[_NON_MONITORING_INTERACTION_KEY] = { + "interaction_kind": interaction_kind, + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": inbound_message_id, + } + if inbound_message_id is not None: + fallback_payload["reply_to_message_id"] = inbound_message_id return await self._send_request("sendMessage", fallback_payload) raise @@ -10048,8 +10990,9 @@ class TelegramGateway: self, lines: list[str], *, - chat_id: str | int | None = None, failure_context: str, + inbound_chat_id: str | int, + inbound_message_id: int, reply_markup: dict | None = None, incident_id: str | None = None, callback_action: str | None = None, @@ -10058,18 +11001,25 @@ class TelegramGateway: ) -> None: """Send a multi-line HTML message without cutting Telegram tags in half.""" chunks = _telegram_html_chunks(lines) - actual_chat_id = str(chat_id or self.alert_chat_id) + actual_chat_id = str(inbound_chat_id) + interaction_context = { + "interaction_kind": "callback_reply", + "inbound_chat_id": actual_chat_id, + "inbound_message_id": inbound_message_id, + } for index, chunk in enumerate(chunks): try: payload: dict = { "chat_id": actual_chat_id, "text": chunk, "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: dict(interaction_context), } source_extra = _callback_reply_source_envelope_extra( incident_id=incident_id, failure_context=failure_context, - status="callback_reply_sent", + status="callback_reply_attempt", chunk_index=index, chunk_count=len(chunks), callback_action=callback_action, @@ -10081,10 +11031,14 @@ class TelegramGateway: payload[_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] = source_extra if index == 0 and reply_markup: payload["reply_markup"] = reply_markup - await self._send_request( + primary_result = await self._send_request( "sendMessage", payload, ) + if not _telegram_send_delivery_succeeded(primary_result): + raise TelegramGatewayError( + "html_callback_reply_delivery_not_proven" + ) except Exception as exc: logger.warning( "telegram_html_line_message_failed", @@ -10096,11 +11050,13 @@ class TelegramGateway: fallback_payload: dict = { "chat_id": actual_chat_id, "text": _plain_text_from_html(chunk), + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: dict(interaction_context), } fallback_source_extra = _callback_reply_source_envelope_extra( incident_id=incident_id, failure_context=failure_context, - status="callback_reply_fallback_sent", + status="callback_reply_fallback_attempt", chunk_index=index, chunk_count=len(chunks), callback_action=callback_action, @@ -10114,10 +11070,14 @@ class TelegramGateway: if index == 0 and reply_markup: fallback_payload["reply_markup"] = reply_markup try: - await self._send_request( + fallback_result = await self._send_request( "sendMessage", fallback_payload, ) + if not _telegram_send_delivery_succeeded(fallback_result): + raise TelegramGatewayError( + "plain_callback_reply_delivery_not_proven" + ) except Exception as fallback_exc: logger.warning( "telegram_html_line_message_plain_fallback_failed", @@ -10129,12 +11089,14 @@ class TelegramGateway: rescue_payload: dict = { "chat_id": actual_chat_id, "text": _plain_text_from_html(chunk, limit=3500), + "reply_to_message_id": inbound_message_id, "_skip_incident_thread_reply": True, + _NON_MONITORING_INTERACTION_KEY: dict(interaction_context), } rescue_source_extra = _callback_reply_source_envelope_extra( incident_id=incident_id, failure_context=failure_context, - status="callback_reply_rescue_sent", + status="callback_reply_rescue_attempt", chunk_index=index, chunk_count=len(chunks), callback_action=callback_action, @@ -10146,10 +11108,14 @@ class TelegramGateway: if rescue_source_extra: rescue_payload[_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] = rescue_source_extra try: - await self._send_request( + rescue_result = await self._send_request( "sendMessage", rescue_payload, ) + if not _telegram_send_delivery_succeeded(rescue_result): + raise TelegramGatewayError( + "rescue_callback_reply_delivery_not_proven" + ) except Exception as rescue_exc: logger.error( "telegram_html_line_message_rescue_failed", @@ -10173,8 +11139,12 @@ class TelegramGateway: text: str, parse_mode: str = "HTML", reply_markup: dict | None = None, + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, ) -> dict: - """發送告警型純文字通知到 SRE 戰情室群組。""" + """Send only when the caller supplies a complete canonical route.""" safe_text, effective_parse_mode = normalize_alert_notification_payload( text, parse_mode, @@ -10184,6 +11154,12 @@ class TelegramGateway: "text": safe_text[:4096], "parse_mode": effective_parse_mode, } + if product_id and signal_family and severity: + payload[_CANONICAL_ROUTE_CONTEXT_KEY] = { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } if reply_markup: payload["reply_markup"] = reply_markup payload[_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] = ( @@ -10195,6 +11171,35 @@ class TelegramGateway: ) return await self._send_request("sendMessage", payload) + async def send_canonical_message( + self, + *, + product_id: str, + signal_family: str, + severity: str, + text: str, + parse_mode: str = "HTML", + reply_markup: dict | None = None, + disable_web_page_preview: bool = True, + reply_to_message_id: int | None = None, + ) -> dict: + """Single production entry point for a classified Telegram message.""" + payload: dict[str, object] = { + "text": text[:4096], + "parse_mode": parse_mode, + "disable_web_page_preview": disable_web_page_preview, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + }, + } + if reply_markup: + payload["reply_markup"] = reply_markup + if reply_to_message_id is not None: + payload["reply_to_message_id"] = reply_to_message_id + return await self._send_request("sendMessage", payload) + # ========================================================================= # 2026-05-04 Claude Sonnet 4.6: send_text 公開 wrapper（修復 drift_adopt_telegram_failed） # ========================================================================= @@ -10205,6 +11210,13 @@ class TelegramGateway: chat_id: int | str | None = None, parse_mode: str = "HTML", disable_web_page_preview: bool = True, + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, + interaction_kind: str | None = None, + inbound_chat_id: str | int | None = None, + inbound_message_id: int | None = None, ) -> dict: """ 公開 send_text wrapper — 委派至 _send_request('sendMessage', ...) @@ -10227,11 +11239,25 @@ class TelegramGateway: parse_mode, ) payload: dict = { - "chat_id": chat_id or self.alert_chat_id, + "chat_id": chat_id or "", "text": safe_text[:4096], "parse_mode": effective_parse_mode, "disable_web_page_preview": disable_web_page_preview, } + if product_id and signal_family and severity: + payload[_CANONICAL_ROUTE_CONTEXT_KEY] = { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } + elif interaction_kind: + payload[_NON_MONITORING_INTERACTION_KEY] = { + "interaction_kind": interaction_kind, + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": inbound_message_id, + } + if inbound_message_id is not None: + payload["reply_to_message_id"] = inbound_message_id return await self._send_request("sendMessage", payload) async def send_controlled_apply_result_receipt( @@ -10253,14 +11279,6 @@ class TelegramGateway: ) -> dict: """Send and mirror the AI Agent controlled-apply result receipt.""" - if not self.alert_chat_id: - logger.warning( - "controlled_apply_result_receipt_skipped_no_chat", - incident_id=incident_id, - apply_op_id=apply_op_id, - ) - return {} - no_write_replay = execution_kind == "no_write_replay" success = ( not no_write_replay @@ -10428,10 +11446,14 @@ class TelegramGateway: f"Runs: {html.escape(incident_runs_url(incident_id, project_id=project_id or 'awoooi'))}", ] payload: dict = { - "chat_id": self.alert_chat_id, "text": "\n".join(lines)[:4096], "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, } reply_markup = incident_truth_chain_reply_markup( incident_id, @@ -10464,6 +11486,11 @@ class TelegramGateway: payload: dict = { "chat_id": chat_id, "text": text[:4096], + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "operator_command_reply", + "inbound_chat_id": str(chat_id), + "inbound_message_id": reply_to_message_id, + }, } if reply_to_message_id: payload["reply_to_message_id"] = reply_to_message_id @@ -10479,9 +11506,14 @@ class TelegramGateway: text: str, parse_mode: str = "HTML", reply_markup: dict | None = None, + *, + legacy_notification_type: str | None = None, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, ) -> dict: """ - 用 @tsenyangbot 發訊息到 SRE 群組 (SRE_GROUP_CHAT_ID) + 透過中央 canonical policy 解析後，才允許發往 SRE 群組。 Args: text: 訊息內容 @@ -10491,15 +11523,18 @@ class TelegramGateway: Returns: dict: Telegram API 回應 (含 message_id) """ - if not settings.SRE_GROUP_CHAT_ID: - logger.warning("send_to_group_skipped", reason="SRE_GROUP_CHAT_ID not configured") - return {} - payload: dict = { - "chat_id": settings.SRE_GROUP_CHAT_ID, "text": text[:4096], "parse_mode": parse_mode, } + if legacy_notification_type: + payload[_LEGACY_NOTIFICATION_TYPE_KEY] = legacy_notification_type + elif product_id and signal_family and severity: + payload[_CANONICAL_ROUTE_CONTEXT_KEY] = { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } if reply_markup: payload["reply_markup"] = reply_markup @@ -10508,17 +11543,21 @@ class TelegramGateway: async def _send_as_bot( self, token: str, - chat_id: str, text: str, reply_to_message_id: int | None = None, parse_mode: str = "HTML", + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, + actual_bot_alias: str, + inbound_chat_id: str | int | None = None, ) -> dict: """ 用指定 Bot Token 發訊息。 Args: token: Bot Token - chat_id: 群組 Chat ID text: 訊息內容 reply_to_message_id: 回覆哪則訊息的 message_id parse_mode: 解析模式 @@ -10526,26 +11565,74 @@ class TelegramGateway: Returns: dict: Telegram API 回應 """ - if not self._http_client: - raise TelegramGatewayError("HTTP client not initialized") - - url = f"{self.TELEGRAM_API_BASE}/bot{token}/sendMessage" - payload: dict = { - "chat_id": chat_id, + route_context = ( + { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } + if product_id and signal_family and severity + else None + ) + candidate_payload: dict[str, object] = { + "chat_id": str(inbound_chat_id or settings.SRE_GROUP_CHAT_ID or ""), "text": text[:4096], "parse_mode": parse_mode, } - # 2026-04-03 ogt: supergroup 跨 Bot reply 需用 reply_parameters (Bot API v6.7+) - # 舊的 reply_to_message_id 在 supergroup 會 400，改用新格式 + allow_sending_without_reply if reply_to_message_id: - payload["reply_parameters"] = { + candidate_payload["reply_parameters"] = { "message_id": reply_to_message_id, "allow_sending_without_reply": True, } + if route_context is not None: + chat_id, route_receipt = self._authorize_canonical_send_message( + payload=candidate_payload, + route_context=route_context, + legacy_notification_type=None, + actual_bot_alias=actual_bot_alias, + ) + elif reply_to_message_id: + chat_id, route_receipt = self._authorize_non_monitoring_interaction( + payload=candidate_payload, + interaction_context={ + "interaction_kind": "bot_discussion_reply", + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": reply_to_message_id, + }, + actual_bot_alias=actual_bot_alias, + ) + else: + chat_id, route_receipt = ( + None, + self._blocked_route_receipt( + "missing_product_signal_severity_or_interaction_context" + ), + ) + if chat_id is None: + return { + "ok": False, + "_awooop_outbound_mirror_acknowledged": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": route_receipt, + } + + if not self._initialized: + await self.initialize() + if not self._http_client: + raise TelegramGatewayError("HTTP client not initialized") + + url = f"{self.TELEGRAM_API_BASE}/bot{token}/sendMessage" + payload = dict(candidate_payload) + payload["chat_id"] = chat_id + response = await self._http_client.post(url, json=payload) response.raise_for_status() result = response.json() + route_receipt["provider_send_performed"] = True + if isinstance(result, dict): + result["_awoooi_canonical_route_receipt"] = route_receipt result_val = result.get("result") if isinstance(result, dict) else None if isinstance(result_val, dict) and "message_id" in result_val: await self._mirror_outbound_message( @@ -10559,6 +11646,11 @@ class TelegramGateway: self, text: str, reply_to_message_id: int | None = None, + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, + inbound_chat_id: str | int | None = None, ) -> dict: """ 用 @OpenClawAwoooI_Bot 在群組發言 @@ -10570,21 +11662,30 @@ class TelegramGateway: Returns: dict: Telegram API 回應 """ - if not settings.OPENCLAW_BOT_TOKEN or not settings.SRE_GROUP_CHAT_ID: - logger.warning("send_as_openclaw_skipped", reason="OPENCLAW_BOT_TOKEN or SRE_GROUP_CHAT_ID not configured") - return {} + if not settings.OPENCLAW_BOT_TOKEN: + logger.warning("send_as_openclaw_skipped", reason="OPENCLAW_BOT_TOKEN not configured") + return self._blocked_route_receipt("bot_binding_unconfigured") return await self._send_as_bot( token=settings.OPENCLAW_BOT_TOKEN, - chat_id=settings.SRE_GROUP_CHAT_ID, text=text, reply_to_message_id=reply_to_message_id, + product_id=product_id, + signal_family=signal_family, + severity=severity, + actual_bot_alias="openclaw_bot", + inbound_chat_id=inbound_chat_id, ) async def send_as_nemotron( self, text: str, reply_to_message_id: int | None = None, + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, + inbound_chat_id: str | int | None = None, ) -> dict: """ 用 @NemoTronAwoooI_Bot 在群組發言 @@ -10596,21 +11697,27 @@ class TelegramGateway: Returns: dict: Telegram API 回應 """ - if not settings.NEMOTRON_BOT_TOKEN or not settings.SRE_GROUP_CHAT_ID: - logger.warning("send_as_nemotron_skipped", reason="NEMOTRON_BOT_TOKEN or SRE_GROUP_CHAT_ID not configured") - return {} + if not settings.NEMOTRON_BOT_TOKEN: + logger.warning("send_as_nemotron_skipped", reason="NEMOTRON_BOT_TOKEN not configured") + return self._blocked_route_receipt("bot_binding_unconfigured") return await self._send_as_bot( token=settings.NEMOTRON_BOT_TOKEN, - chat_id=settings.SRE_GROUP_CHAT_ID, text=text, reply_to_message_id=reply_to_message_id, + product_id=product_id, + signal_family=signal_family, + severity=severity, + actual_bot_alias="nemotron_bot", + inbound_chat_id=inbound_chat_id, ) async def trigger_group_ai_discussion( self, alert_message_id: int, alert_summary: str, + *, + inbound_chat_id: str | int | None = None, ) -> None: """ 觸發群組 AI 並行分析（三頭政治核心流程） @@ -10626,6 +11733,13 @@ class TelegramGateway: alert_message_id: 告警訊息的 message_id（兩個 Bot 回覆的起點） alert_summary: 告警摘要文字（提供給 AI 分析用） """ + if not inbound_chat_id: + logger.warning( + "trigger_group_ai_discussion_missing_verified_inbound_no_send", + alert_message_id=alert_message_id, + ) + return + try: from src.services.chat_manager import ChatManager # noqa: PLC0415 except ImportError: @@ -10649,11 +11763,18 @@ class TelegramGateway: ) if openclaw_analysis and not isinstance(openclaw_analysis, Exception): - await self.send_as_openclaw( + result = await self.send_as_openclaw( text=f"🦞 OpenClaw 分析\n\n{openclaw_analysis}", reply_to_message_id=alert_message_id, + inbound_chat_id=inbound_chat_id, ) - logger.info("group_ai_discussion_openclaw_sent") + if _telegram_send_delivery_succeeded(result): + logger.info("group_ai_discussion_openclaw_sent") + else: + logger.warning( + "group_ai_discussion_openclaw_blocked_no_send", + alert_message_id=alert_message_id, + ) else: logger.warning("trigger_group_ai_discussion_openclaw_empty") @@ -10998,6 +12119,7 @@ class TelegramGateway: message_id=message_id, original_text=original_text, username=username, + chat_id=chat_id, ) if result.get("success"): @@ -11008,6 +12130,7 @@ class TelegramGateway: user_id=user_id, username=username, message_id=message_id, + inbound_chat_id=chat_id, ) async def _handle_chat_message(self, update_id: int, message: dict) -> None: @@ -11034,6 +12157,7 @@ class TelegramGateway: await svc.download_and_analyze( chat_id=str(chat_id), file_id=file_id, + original_message_id=message_id, question=caption, ) except Exception as _img_err: @@ -11079,11 +12203,25 @@ class TelegramGateway: whitelist = settings.get_tg_user_whitelist() if not whitelist or user_id not in whitelist: logger.warning("telegram_ai_command_unauthorized", user_id=user_id, whitelist_empty=not whitelist) - await self.send_notification("⛔ 未授權：/ai 指令僅限白名單用戶", parse_mode="HTML", chat_id=chat_id) + await self.send_notification( + "⛔ 未授權：/ai 指令僅限白名單用戶", + parse_mode="HTML", + chat_id=chat_id, + interaction_kind="operator_command_reply", + inbound_chat_id=chat_id, + inbound_message_id=message_id, + ) return from src.services.ai_control import handle_ai_command response = await handle_ai_command(text.strip()) - await self.send_notification(response, parse_mode="HTML", chat_id=chat_id) + await self.send_notification( + response, + parse_mode="HTML", + chat_id=chat_id, + interaction_kind="operator_command_reply", + inbound_chat_id=chat_id, + inbound_message_id=message_id, + ) logger.info("telegram_ai_command_handled", user_id=user_id, text=text[:50]) return @@ -11097,7 +12235,14 @@ class TelegramGateway: username=username, message_text=text, ) - await self.send_notification(response, parse_mode="HTML", chat_id=chat_id) + await self.send_notification( + response, + parse_mode="HTML", + chat_id=chat_id, + interaction_kind="operator_command_reply", + inbound_chat_id=chat_id, + inbound_message_id=message_id, + ) async def _handle_group_message( self, @@ -11213,6 +12358,7 @@ class TelegramGateway: await self.send_as_openclaw( text=f"🦞 OpenClaw\n\n{body}", reply_to_message_id=message_id, + inbound_chat_id=chat_id, ) elif mention_nemo and not mention_openclaw: @@ -11225,6 +12371,7 @@ class TelegramGateway: await self.send_as_nemotron( text=f"🤖 NemoClaw\n\n{body}", reply_to_message_id=message_id, + inbound_chat_id=chat_id, ) else: @@ -11241,6 +12388,7 @@ class TelegramGateway: await self.send_as_openclaw( text=f"🦞 OpenClaw\n\n{_clean_ai_reply(oc_result)}", reply_to_message_id=message_id, + inbound_chat_id=chat_id, ) if nemo_result and not isinstance(nemo_result, Exception): @@ -11248,6 +12396,7 @@ class TelegramGateway: await self.send_as_nemotron( text=f"🤖 NemoClaw\n\n{nemo_body}", reply_to_message_id=message_id, + inbound_chat_id=chat_id, ) logger.info("group_message_handled", user_id=user_id, text=text[:50]) @@ -11284,7 +12433,11 @@ class TelegramGateway: msg = "\n".join(lines) except Exception as e: msg = f"🖥 Cluster 狀態\n⚠️ 無法取得: {e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/incidents": try: @@ -11299,7 +12452,11 @@ class TelegramGateway: msg = "🚨 活躍告警\n✅ 目前無告警" except Exception as e: msg = f"🚨 活躍告警\n⚠️ 無法取得: {e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/cost": redis = get_redis() @@ -11316,7 +12473,11 @@ class TelegramGateway: ) except Exception as e: msg = f"💰 費用統計\n⚠️ 無法取得: {e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/pods": try: @@ -11332,7 +12493,11 @@ class TelegramGateway: msg = "⚠️ 異常 Pod\n✅ 全部 Pod 正常" except Exception as e: msg = f"⚠️ 異常 Pod\n⚠️ 無法取得: {e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/rag": # /rag <查詢內容> — RAG 知識庫語義查詢 (ADR-067 Phase 33) @@ -11342,12 +12507,14 @@ class TelegramGateway: await self.send_as_openclaw( text="📚 RAG 知識庫查詢\n用法: /rag 你的問題\n例如: /rag 什麼是 ADR-067？", reply_to_message_id=message_id, + inbound_chat_id=_chat_id, ) return question = parts[1].strip() await self.send_as_openclaw( text=f"📚 查詢知識庫中...\n{question[:80]}", reply_to_message_id=message_id, + inbound_chat_id=_chat_id, ) try: from src.services.knowledge_rag_service import get_knowledge_rag_service @@ -11357,7 +12524,11 @@ class TelegramGateway: except Exception as e: logger.warning("rag_telegram_query_failed", error=str(e)) msg = f"📚 RAG 查詢失敗\n{e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/help": msg = ( @@ -11374,7 +12545,11 @@ class TelegramGateway: "• 小賀 或 @NemoTronAwoooI_Bot → 只有 NemoClaw\n" "• Reply 某個 Bot 的訊息 → 只有那個 Bot 回應" ) - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) else: logger.debug("group_unknown_command", cmd=cmd) @@ -11394,6 +12569,8 @@ class TelegramGateway: action: str, username: str, execution_triggered: bool, + inbound_chat_id: str | int, + inbound_message_id: int, approval_action: str | None = None, repair_candidate_work_item_href: str = "", repair_candidate_work_item_id: str = "", @@ -11404,30 +12581,47 @@ class TelegramGateway: 策略： 1. editMessageReplyMarkup — 移除批准/拒絕按鈕，保留資訊按鈕 2. sendMessage reply_to → 在原訊息下方附加狀態行 - 3. 如果 message_id 找不到，fallback 到 send_notification + 3. HTML 回覆失敗只可對同一 inbound parent 重試純文字 + 4. 重試仍失敗寫 durable callback-failure receipt，不另發共享 SRE """ import html as _html - chat_id = self.alert_chat_id - if not chat_id: - return + chat_id = str(inbound_chat_id or "") + try: + exact_parent_message_id = int(inbound_message_id) + except (TypeError, ValueError): + exact_parent_message_id = 0 + try: + supplied_message_id = int(message_id) if message_id is not None else 0 + except (TypeError, ValueError): + supplied_message_id = 0 - # 找到原始告警訊息 ID（優先 Redis，fallback DB） - orig_msg_id = message_id - if not orig_msg_id: - try: - redis = await get_redis() - _val = await redis.get(f"tg_msg:{incident_id}") - if _val: - orig_msg_id = int(_val) - else: - # DB fallback - from src.services.approval_db import get_approval_service as _svc - _approvals = await _svc().get_all_approvals(incident_id=incident_id) - if _approvals and _approvals[0].telegram_message_id: - orig_msg_id = _approvals[0].telegram_message_id - except Exception: - pass + if ( + not chat_id + or exact_parent_message_id <= 0 + or ( + supplied_message_id > 0 + and supplied_message_id != exact_parent_message_id + ) + ): + mismatch_error = TelegramGatewayError( + "missing_or_mismatched_verified_approval_callback_parent" + ) + await self._record_callback_reply_failure( + chat_id=chat_id, + incident_id=incident_id, + failure_context="approval_result_reply", + callback_action=action, + chunk_index=0, + chunk_count=1, + error=mismatch_error, + parent_message_id=( + exact_parent_message_id + if exact_parent_message_id > 0 + else None + ), + ) + return if action == "approve": status_emoji = "✅" @@ -11458,71 +12652,133 @@ class TelegramGateway: status_line = f"{status_emoji} {status_text}　{suffix}".strip() - if orig_msg_id: - try: - # 1. 移除批准/拒絕按鈕（只保留資訊按鈕列） - if no_action_approval: - no_action_first_row: list[dict[str, str]] = [] - work_item_deep_link = _repair_candidate_work_item_url( - incident_id=incident_id, - repair_candidate_work_item_href=repair_candidate_work_item_href, - repair_candidate_work_item_id=repair_candidate_work_item_id, - ) - if work_item_deep_link: - no_action_first_row.append({ - "text": "🧾 Work Item", - "url": work_item_deep_link, - }) - no_action_first_row.append( - {"text": "🧰 處置包", "callback_data": f"detail:{incident_id}"} - ) - info_buttons = [ - no_action_first_row, - [ - {"text": "🔄 重診", "callback_data": f"reanalyze:{incident_id}"}, - {"text": "📊 歷史", "callback_data": f"history:{incident_id}"}, - ], - ] - else: - info_buttons = [[ - {"text": "📋 詳情", "callback_data": f"detail:{incident_id}"}, - {"text": "📊 歷史", "callback_data": f"history:{incident_id}"}, - ]] - awooop_row = _awooop_truth_chain_button_row(incident_id) - if awooop_row: - info_buttons.append(awooop_row) - await self._send_request( - "editMessageReplyMarkup", - { - "chat_id": chat_id, - "message_id": orig_msg_id, - "reply_markup": {"inline_keyboard": info_buttons}, - }, - ) - except Exception as _e: - # 2026-04-09 Claude Sonnet 4.6: I3 架構Review修復 — 加 warning 防止靜默失敗 - logger.warning("notify_approval_edit_keyboard_failed", incident_id=incident_id, error=str(_e)) - - try: - # 2. 在原訊息下回覆狀態 - await self._send_request( - "sendMessage", - { - "chat_id": chat_id, - "text": status_line, - "parse_mode": "HTML", - "reply_to_message_id": orig_msg_id, - }, - ) - return - except Exception as _e: - logger.warning("notify_approval_reply_failed", incident_id=incident_id, error=str(_e)) - - # fallback: 發新通知 try: - await self.send_alert_notification(status_line, parse_mode="HTML") + # 1. 移除批准/拒絕按鈕（只保留資訊按鈕列） + if no_action_approval: + no_action_first_row: list[dict[str, str]] = [] + work_item_deep_link = _repair_candidate_work_item_url( + incident_id=incident_id, + repair_candidate_work_item_href=repair_candidate_work_item_href, + repair_candidate_work_item_id=repair_candidate_work_item_id, + ) + if work_item_deep_link: + no_action_first_row.append({ + "text": "🧾 Work Item", + "url": work_item_deep_link, + }) + no_action_first_row.append( + {"text": "🧰 處置包", "callback_data": f"detail:{incident_id}"} + ) + info_buttons = [ + no_action_first_row, + [ + {"text": "🔄 重診", "callback_data": f"reanalyze:{incident_id}"}, + {"text": "📊 歷史", "callback_data": f"history:{incident_id}"}, + ], + ] + else: + info_buttons = [[ + {"text": "📋 詳情", "callback_data": f"detail:{incident_id}"}, + {"text": "📊 歷史", "callback_data": f"history:{incident_id}"}, + ]] + awooop_row = _awooop_truth_chain_button_row(incident_id) + if awooop_row: + info_buttons.append(awooop_row) + await self._send_request( + "editMessageReplyMarkup", + { + "chat_id": chat_id, + "message_id": exact_parent_message_id, + "reply_markup": {"inline_keyboard": info_buttons}, + }, + ) except Exception as _e: - logger.warning("notify_approval_fallback_failed", incident_id=incident_id, error=str(_e)) + logger.warning( + "notify_approval_edit_keyboard_failed", + incident_id=incident_id, + error=str(_e), + ) + + interaction_context = { + "interaction_kind": "callback_reply", + "inbound_chat_id": chat_id, + "inbound_message_id": exact_parent_message_id, + } + primary_source_extra = _callback_reply_source_envelope_extra( + incident_id=incident_id, + failure_context="approval_result_reply", + status="callback_reply_sent", + chunk_index=0, + chunk_count=1, + callback_action=action, + parse_mode="HTML", + parent_message_id=exact_parent_message_id, + ) + try: + primary_result = await self._send_request( + "sendMessage", + { + "chat_id": chat_id, + "text": status_line, + "parse_mode": "HTML", + "reply_to_message_id": exact_parent_message_id, + _NON_MONITORING_INTERACTION_KEY: interaction_context, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: primary_source_extra, + }, + ) + if not _telegram_send_delivery_succeeded(primary_result): + raise TelegramGatewayError( + "approval_callback_html_delivery_not_proven" + ) + return + except Exception as primary_error: + logger.warning( + "notify_approval_reply_failed", + incident_id=incident_id, + error=str(primary_error), + ) + + fallback_source_extra = _callback_reply_source_envelope_extra( + incident_id=incident_id, + failure_context="approval_result_reply", + status="callback_reply_fallback_sent", + chunk_index=0, + chunk_count=1, + callback_action=action, + parent_message_id=exact_parent_message_id, + ) + try: + fallback_result = await self._send_request( + "sendMessage", + { + "chat_id": chat_id, + "text": _plain_text_from_html(status_line, limit=500), + "reply_to_message_id": exact_parent_message_id, + _NON_MONITORING_INTERACTION_KEY: interaction_context, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: fallback_source_extra, + }, + ) + if not _telegram_send_delivery_succeeded(fallback_result): + raise TelegramGatewayError( + "approval_callback_plain_delivery_not_proven" + ) + return + except Exception as fallback_error: + logger.warning( + "notify_approval_exact_fallback_failed_no_send", + incident_id=incident_id, + error=str(fallback_error), + ) + await self._record_callback_reply_failure( + chat_id=chat_id, + incident_id=incident_id, + failure_context="approval_result_reply", + callback_action=action, + chunk_index=0, + chunk_count=1, + error=fallback_error, + parent_message_id=exact_parent_message_id, + ) async def _execute_approval_action( self, @@ -11531,6 +12787,7 @@ class TelegramGateway: user_id: int, username: str, message_id: int, # noqa: ARG002 + inbound_chat_id: str | int | None = None, ) -> None: """ 執行簽核動作 (更新資料庫) @@ -11603,6 +12860,8 @@ class TelegramGateway: action="approve", username=username, execution_triggered=execution_triggered and _execution_allowed, + inbound_chat_id=inbound_chat_id or "", + inbound_message_id=message_id, approval_action=getattr(approval, "action", None), repair_candidate_work_item_href=_work_item_href, repair_candidate_work_item_id=_work_item_id, @@ -11617,6 +12876,8 @@ class TelegramGateway: action="approve", username=username, execution_triggered=False, + inbound_chat_id=inbound_chat_id or "", + inbound_message_id=message_id, approval_action=getattr(approval, "action", None), repair_candidate_work_item_href=_work_item_href, repair_candidate_work_item_id=_work_item_id, @@ -11625,9 +12886,14 @@ class TelegramGateway: # 告警已是 execution_failed / execution_success / rejected 等終態 try: await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id or "", "text": f"ℹ️ 此告警已處理（狀態：{status_val}），無法重複批准 by @{username}", "reply_to_message_id": message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": message_id, + }, }) except Exception as _ne: logger.warning("telegram_approval_already_resolved_notify_failed", error=str(_ne)) @@ -11692,6 +12958,8 @@ class TelegramGateway: action="reject", username=username, execution_triggered=False, + inbound_chat_id=inbound_chat_id or "", + inbound_message_id=message_id, ) try: from src.services.incident_approval_service import ( @@ -11859,21 +13127,31 @@ class TelegramGateway: text = report_to_telegram_html(report) - # 只發到 AwoooI SRE 戰情室；缺設定時不得旁路到個人或其他群組。 - if not settings.SRE_GROUP_CHAT_ID: - logger.warning( - "telegram_heartbeat_skipped", - reason="SRE_GROUP_CHAT_ID not configured", + # 警告態屬 shared infrastructure P1；健康 heartbeat 不得進 SRE。 + route_result = await self.send_to_group( + text=text, + product_id="awoooi", + signal_family=( + "shared_infrastructure" + if report.warnings + else "product_raw_monitoring" + ), + severity="P1" if report.warnings else "P3", + ) + if not _telegram_send_delivery_succeeded(route_result): + logger.info( + "telegram_heartbeat_policy_blocked", + warnings=len(report.warnings), + delivery_status=route_result.get("_awooop_delivery_status"), ) return False - await self.send_to_group(text=text) self._last_message_time = datetime.now(UTC) logger.info( - "telegram_heartbeat_sent", + "telegram_heartbeat_routed", warnings=len(report.warnings), warnings_hash=warnings_hash, - has_sre_group=bool(settings.SRE_GROUP_CHAT_ID), + delivery_status=route_result.get("_awooop_delivery_status", "sent"), ) return True diff --git a/apps/api/src/services/weekly_report_service.py b/apps/api/src/services/weekly_report_service.py index ea959912b..19155c0a7 100644 --- a/apps/api/src/services/weekly_report_service.py +++ b/apps/api/src/services/weekly_report_service.py @@ -24,17 +24,21 @@ import json import subprocess from datetime import datetime, timedelta from pathlib import Path +from typing import Protocol, runtime_checkable from urllib.parse import quote, urlencode from urllib.request import Request, urlopen -from typing import Protocol, runtime_checkable - -import structlog from zoneinfo import ZoneInfo +import structlog + from src.core.config import settings -from src.services.stats_service import StatsService, get_stats_service from src.services.k3s_monitor_service import K3sMonitorService, get_k3s_monitor_service -from src.services.telegram_gateway import WeeklyReportMessage, get_telegram_gateway +from src.services.stats_service import StatsService, get_stats_service +from src.services.telegram_gateway import ( + WeeklyReportMessage, + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) logger = structlog.get_logger(__name__) @@ -449,14 +453,17 @@ class WeeklyReportService: # 取得 Telegram Gateway gateway = get_telegram_gateway() - if not gateway._initialized: - await gateway.initialize() # 發送訊息 formatted = report.format() - result = await gateway.send_text(formatted) + result = await gateway.send_text( + formatted, + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) - if result: + if _telegram_send_delivery_succeeded(result): logger.info("weekly_report_sent", week=report.week_range) return True else: diff --git a/apps/api/tests/test_alert_converged_recurrence.py b/apps/api/tests/test_alert_converged_recurrence.py index 5bab17725..ed2863f61 100644 --- a/apps/api/tests/test_alert_converged_recurrence.py +++ b/apps/api/tests/test_alert_converged_recurrence.py @@ -20,8 +20,8 @@ class _FakeGateway: self.primary_messages = [] self.private_messages = [] - async def send_alert_notification(self, text): - self.primary_messages.append(text) + async def send_alert_notification(self, text, **route): + self.primary_messages.append({"text": text, "route": route}) return {"ok": True} async def send_notification(self, text, *, chat_id=None): @@ -150,4 +150,9 @@ async def test_converged_recurrence_does_not_mirror_to_private_chat(monkeypatch) ) assert len(gateway.primary_messages) == 1 + assert gateway.primary_messages[0]["route"] == { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + } assert gateway.private_messages == [] diff --git a/apps/api/tests/test_ansible_verified_closure.py b/apps/api/tests/test_ansible_verified_closure.py index de64fff4a..f43b2eff5 100644 --- a/apps/api/tests/test_ansible_verified_closure.py +++ b/apps/api/tests/test_ansible_verified_closure.py @@ -11,6 +11,7 @@ import pytest from src.core import redis_client as redis_client_module from src.jobs import awooop_ansible_candidate_backfill_job as candidate_job from src.services import awooop_ansible_check_mode_service as service +from src.services import telegram_gateway as telegram_gateway_module from src.services.telegram_gateway import TelegramGateway from src.workers import ansible_executor_broker as broker @@ -89,6 +90,11 @@ def _controlled_result_payload() -> dict: return { "chat_id": "sre-chat", "text": "CONTROLLED APPLY RESULT INC-20260711-D037E5", + "_awoooi_canonical_telegram_route": { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, "_skip_incident_thread_reply": True, "_awooop_source_envelope_extra": { "callback_reply": { @@ -749,6 +755,11 @@ async def test_controlled_result_pending_reservation_never_resends_provider( return _Response() gateway = TelegramGateway() + monkeypatch.setattr( + telegram_gateway_module.settings, + "SRE_GROUP_CHAT_ID", + "sre-chat", + ) gateway._initialized = True client = _Client() gateway._http_client = client @@ -801,6 +812,11 @@ async def test_controlled_result_reuses_durable_sent_reservation( monkeypatch: pytest.MonkeyPatch, ) -> None: gateway = TelegramGateway() + monkeypatch.setattr( + telegram_gateway_module.settings, + "SRE_GROUP_CHAT_ID", + "sre-chat", + ) gateway._initialized = True client = AsyncMock() gateway._http_client = client diff --git a/apps/api/tests/test_channel_hub_grouped_alert_events.py b/apps/api/tests/test_channel_hub_grouped_alert_events.py index 9f874ff5a..17d10478e 100644 --- a/apps/api/tests/test_channel_hub_grouped_alert_events.py +++ b/apps/api/tests/test_channel_hub_grouped_alert_events.py @@ -21,6 +21,8 @@ from src.services.channel_hub import ( record_outbound_message, ) +_FAKE_SECRET_SHAPED_VALUE = "1234567890:" + "abcdefghijklmnopqrstuvwxyzABCDEFGH" + def test_db_timestamp_now_is_naive_utc_for_asyncpg() -> None: assert _db_timestamp_now().tzinfo is None @@ -37,6 +39,9 @@ async def test_send_telegram_interim_uses_gateway_mirror_path(monkeypatch) -> No chat_id: str | int | None = None, parse_mode: str = "HTML", disable_web_page_preview: bool = True, + interaction_kind: str | None = None, + inbound_chat_id: str | int | None = None, + inbound_message_id: int | None = None, ) -> dict[str, object]: calls.append( { @@ -44,31 +49,91 @@ async def test_send_telegram_interim_uses_gateway_mirror_path(monkeypatch) -> No "chat_id": chat_id, "parse_mode": parse_mode, "disable_web_page_preview": disable_web_page_preview, + "interaction_kind": interaction_kind, + "inbound_chat_id": inbound_chat_id, + "inbound_message_id": inbound_message_id, } ) - return {"ok": True, "result": {"message_id": 42}} + return { + "ok": True, + "result": {"message_id": 42}, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + } monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: _FakeGateway(), ) - await _send_telegram_interim( + delivered = await _send_telegram_interim( channel_chat_id="123456", content="AI 正在分析中，請稍候...", run_id=uuid4(), + inbound_message_id=321, ) + assert delivered is True assert calls == [ { "text": "AI 正在分析中，請稍候...", "chat_id": "123456", "parse_mode": "HTML", "disable_web_page_preview": True, + "interaction_kind": "operator_command_reply", + "inbound_chat_id": "123456", + "inbound_message_id": 321, } ] +async def test_send_telegram_interim_fails_closed_without_verified_parent( + monkeypatch, +) -> None: + class _UnexpectedGateway: + async def send_text(self, *args, **kwargs): + raise AssertionError("gateway must not be called") + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: _UnexpectedGateway(), + ) + + delivered = await _send_telegram_interim( + channel_chat_id="123456", + content="must remain no-send", + run_id=uuid4(), + ) + + assert delivered is False + + +async def test_send_telegram_interim_rejects_structured_no_send( + monkeypatch, +) -> None: + class _BlockedGateway: + async def send_text(self, *args, **kwargs): + return { + "ok": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + } + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: _BlockedGateway(), + ) + + delivered = await _send_telegram_interim( + channel_chat_id="123456", + content="must remain no-send", + run_id=uuid4(), + inbound_message_id=321, + ) + + assert delivered is False + + def test_channel_hub_does_not_embed_direct_telegram_bot_api() -> None: source = inspect.getsource(_send_telegram_interim) assert "TELEGRAM_BOT_TOKEN" not in source @@ -106,7 +171,10 @@ def test_build_alertmanager_provider_event_id_keeps_fingerprint() -> None: "incident_linked", ) - assert event_id == "alertmanager:incident_linked:alert-20260513213000:abcdef1234567890abcdef1234567890" + assert event_id == ( + "alertmanager:incident_linked:alert-20260513213000:" + + "abcdef1234567890abcdef1234567890" + ) assert len(event_id) < 256 @@ -142,7 +210,10 @@ def test_build_inbound_source_envelope_redacts_and_keeps_refs() -> None: stage="received", provider_event_id="alertmanager:received:alert-1:fp-1", raw_event_id="alert-1", - raw_content="ACTION REQUIRED INC-20260513-ABCDEF token 1234567890:abcdefghijklmnopqrstuvwxyzABCDEFGH", + raw_content=( + "ACTION REQUIRED INC-20260513-ABCDEF token " + + _FAKE_SECRET_SHAPED_VALUE + ), alertname="DockerContainerUnhealthy", severity="warning", namespace="default", @@ -272,7 +343,7 @@ async def test_mirror_inbound_event_writes_redacted_replay_fields() -> None: stage="received", provider_event_id="sentry:received:123", raw_event_id="123", - raw_content="Sentry token 1234567890:abcdefghijklmnopqrstuvwxyzABCDEFGH", + raw_content="Sentry token " + _FAKE_SECRET_SHAPED_VALUE, alertname="Sentry issue", severity="error", namespace="sentry", @@ -285,7 +356,7 @@ async def test_mirror_inbound_event_writes_redacted_replay_fields() -> None: channel_type="internal", provider_event_id="sentry:received:123", platform_subject_id="sentry", - raw_content="Sentry token 1234567890:abcdefghijklmnopqrstuvwxyzABCDEFGH", + raw_content="Sentry token " + _FAKE_SECRET_SHAPED_VALUE, source_envelope=envelope, ) diff --git a/apps/api/tests/test_failover_alerter.py b/apps/api/tests/test_failover_alerter.py index e8ec893bf..f7f69a3cb 100644 --- a/apps/api/tests/test_failover_alerter.py +++ b/apps/api/tests/test_failover_alerter.py @@ -46,16 +46,15 @@ def mock_redis(): @pytest.fixture def mock_telegram_send(): - """Mock TelegramGateway.send_alert_notification + settings.SRE_GROUP_CHAT_ID + """Mock canonical TelegramGateway send and destination resolution. `_send()` 在函式內 inline import，必須 mock 來源 module 而非 alerter module。 """ - with patch("src.services.telegram_gateway.get_telegram_gateway") as mock_gw, \ - patch("src.core.config.get_settings") as mock_settings: + with patch("src.services.telegram_gateway.get_telegram_gateway") as mock_gw: gateway = MagicMock() gateway.send_alert_notification = AsyncMock() + gateway.canonical_destination_chat_id.return_value = "sre-room" mock_gw.return_value = gateway - mock_settings.return_value = MagicMock(SRE_GROUP_CHAT_ID="-100123", OPENCLAW_TG_CHAT_ID="-100456") yield gateway @@ -117,12 +116,11 @@ async def test_alert_recovery_send(mock_redis, mock_telegram_send): async def test_no_telegram_chat_id_noop(mock_redis): alerter = FailoverAlerter(redis_client=mock_redis) - with patch("src.services.telegram_gateway.get_telegram_gateway") as mock_gw, \ - patch("src.core.config.get_settings") as mock_settings: + with patch("src.services.telegram_gateway.get_telegram_gateway") as mock_gw: gateway = MagicMock() gateway.send_alert_notification = AsyncMock() + gateway.canonical_destination_chat_id.return_value = "" mock_gw.return_value = gateway - mock_settings.return_value = MagicMock(SRE_GROUP_CHAT_ID=None, OPENCLAW_TG_CHAT_ID=None) # 不該 raise，dedup pass 但 send 因 chat_id 缺直接 return await alerter.alert_failover({"to_provider": "gemini"}) @@ -244,11 +242,12 @@ def test_lines_from_list_escapes_markdown_v2_numbered_periods() -> None: def test_sanitize_telegram_error_redacts_bot_token_url() -> None: - raw = "HTTP error for https://api.telegram.org/bot123456:SECRET/sendMessage" + placeholder = "placeholder-value" + raw = "HTTP error for https://api.telegram.org/" + f"bot{placeholder}/sendMessage" sanitized = _sanitize_telegram_error(raw) - assert "SECRET" not in sanitized + assert placeholder not in sanitized assert "bot" in sanitized diff --git a/apps/api/tests/test_notification_matrix_group_cutover.py b/apps/api/tests/test_notification_matrix_group_cutover.py index ca3a6b8d4..cff38cd53 100644 --- a/apps/api/tests/test_notification_matrix_group_cutover.py +++ b/apps/api/tests/test_notification_matrix_group_cutover.py @@ -1,31 +1,56 @@ from src.services.notification_matrix import resolve_chat_ids -def test_all_alert_types_resolve_to_sre_group_first() -> None: +def test_only_shared_incident_types_resolve_to_sre_group() -> None: for notification_type in [ - "TYPE-1", "TYPE-2", "TYPE-3", "TYPE-4", "TYPE-4D", "TYPE-5S", - "TYPE-6B", "TYPE-7E", "TYPE-8M", - "UNKNOWN", ]: assert resolve_chat_ids( notification_type, - dm_chat_id="5619078117", - group_chat_id="-1003711974679", + dm_chat_id="operator-dm-id", + group_chat_id="sre-room-id", tg_group_cutover=False, - ) == ["-1003711974679"] + ) == ["sre-room-id"] + + +def test_info_business_and_unknown_types_are_fail_closed() -> None: + for notification_type in ["TYPE-1", "TYPE-6B", "UNKNOWN"]: + assert ( + resolve_chat_ids( + notification_type, + dm_chat_id="operator-dm-id", + group_chat_id="sre-room-id", + tg_group_cutover=False, + ) + == [] + ) + + +def test_explicit_empty_registry_does_not_fall_back_to_default() -> None: + assert ( + resolve_chat_ids( + "TYPE-5S", + dm_chat_id="operator-dm-id", + group_chat_id="sre-room-id", + registry={}, + ) + == [] + ) def test_dm_is_not_used_when_group_missing() -> None: - assert resolve_chat_ids( - "TYPE-3", - dm_chat_id="5619078117", - group_chat_id="", - tg_group_cutover=True, - ) == [] + assert ( + resolve_chat_ids( + "TYPE-3", + dm_chat_id="operator-dm-id", + group_chat_id="", + tg_group_cutover=True, + ) + == [] + ) diff --git a/apps/api/tests/test_report_generation_service.py b/apps/api/tests/test_report_generation_service.py index 087410929..47b5511c2 100644 --- a/apps/api/tests/test_report_generation_service.py +++ b/apps/api/tests/test_report_generation_service.py @@ -618,8 +618,19 @@ class TestTriggerPostmortemPersistence: op_logs: list[dict] = [] class FakeGateway: - async def send_to_group(self, text: str, parse_mode: str = "HTML") -> None: + async def send_to_group( + self, + text: str, + parse_mode: str = "HTML", + **route, + ) -> dict: sent_messages.append(text) + assert route == { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + } + return {"ok": True} class FakeKnowledgeRepo: def __init__(self, _db) -> None: diff --git a/apps/api/tests/test_telegram_button_consistency.py b/apps/api/tests/test_telegram_button_consistency.py index f96cad5f2..6e0931b00 100644 --- a/apps/api/tests/test_telegram_button_consistency.py +++ b/apps/api/tests/test_telegram_button_consistency.py @@ -179,8 +179,9 @@ class TestSREGroupCutover: ) assert match, "找不到 send_approval_card 函式" fn_body = match.group(0) - assert '"chat_id": target_chat_id' in fn_body - assert "target_chat_id = self.alert_chat_id" in fn_body + assert "_LEGACY_NOTIFICATION_TYPE_KEY" in fn_body + assert 'notification_type or "UNKNOWN"' in fn_body + assert '"chat_id": target_chat_id' not in fn_body assert '"chat_id": self.chat_id' not in fn_body def test_send_approval_card_does_not_double_send_group(self): diff --git a/apps/api/tests/test_telegram_callback_context_routing.py b/apps/api/tests/test_telegram_callback_context_routing.py new file mode 100644 index 000000000..3eca42751 --- /dev/null +++ b/apps/api/tests/test_telegram_callback_context_routing.py @@ -0,0 +1,721 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +import src.repositories.drift_repository as drift_repository +import src.repositories.incident_repository as incident_repository +import src.services.ai_advisory_helpers as ai_advisory_helpers +import src.services.approval_db as approval_db +import src.services.callback_dispatcher as callback_dispatcher +import src.services.telegram_gateway as gateway_module +from src.models.approval import ApprovalStatus +from src.models.incident import IncidentStatus +from src.services.callback_dispatcher import DispatchResult +from src.services.telegram_gateway import TelegramGateway + +_CHAT_ID = "verified-chat" +_MESSAGE_ID = 77 + + +def _assert_verified_callback_payload(payload: dict) -> None: + assert payload["chat_id"] == _CHAT_ID + assert payload["reply_to_message_id"] == _MESSAGE_ID + assert payload[gateway_module._NON_MONITORING_INTERACTION_KEY] == { + "interaction_kind": "callback_reply", + "inbound_chat_id": _CHAT_ID, + "inbound_message_id": _MESSAGE_ID, + } + + +class _DriftRepo: + def __init__(self, value=None, error: Exception | None = None) -> None: + self.value = value + self.error = error + + async def get_by_id(self, _report_id: str): + if self.error: + raise self.error + return self.value + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "repo", + [ + _DriftRepo(), + _DriftRepo(SimpleNamespace(items=[], namespace="awoooi-prod")), + _DriftRepo(error=RuntimeError("read failed")), + ], +) +async def test_drift_detail_page_and_error_keep_verified_callback_context( + monkeypatch: pytest.MonkeyPatch, + repo: _DriftRepo, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(drift_repository, "get_drift_repository", lambda: repo) + + await gateway._send_drift_diff_detail( + "drift-1", + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_request.assert_awaited_once() + method, payload = send_request.await_args.args + assert method == "sendMessage" + _assert_verified_callback_payload(payload) + + +@pytest.mark.asyncio +async def test_advisory_and_drift_outcome_keep_verified_callback_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + monkeypatch.setattr( + ai_advisory_helpers, + "handle_ai_advisory_callback", + AsyncMock(return_value={"feedback_text": "done", "success": True}), + ) + + await gateway._handle_ai_advisory_action( + action="ai_advisory_view", + advisory_payload="capacity:item-1", + callback_query_id="callback-1", + user_id=42, + username="operator", + user={"id": 42}, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + _assert_verified_callback_payload(send_request.await_args.args[1]) + + send_request.reset_mock() + + class _Redis: + async def get(self, _key: str) -> str: + return str(_MESSAGE_ID) + + monkeypatch.setattr(gateway_module, "get_redis", lambda: _Redis()) + await gateway._edit_drift_card_outcome( + report_id="drift-1", + verb="adopted", + by="operator", + ok=True, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + send_payload = next( + call.args[1] + for call in send_request.await_args_list + if call.args[0] == "sendMessage" + ) + _assert_verified_callback_payload(send_payload) + + +@pytest.mark.asyncio +async def test_drift_outcome_ignores_stale_redis_parent_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + redis_reads: list[str] = [] + + class _Redis: + async def get(self, key: str) -> str: + redis_reads.append(key) + return "999" + + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway_module, "get_redis", lambda: _Redis()) + + await gateway._edit_drift_card_outcome( + report_id="drift-stale-parent", + verb="adopted", + by="operator", + ok=True, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + assert redis_reads == ["tg_drift:drift-stale-parent"] + assert [call.args[0] for call in send_request.await_args_list] == [ + "editMessageReplyMarkup", + "sendMessage", + ] + edit_payload = send_request.await_args_list[0].args[1] + assert edit_payload["chat_id"] == _CHAT_ID + assert edit_payload["message_id"] == _MESSAGE_ID + send_payload = send_request.await_args_list[1].args[1] + _assert_verified_callback_payload(send_payload) + for call in send_request.await_args_list: + provider_payload = call.args[1] + assert provider_payload.get("message_id") != 999 + assert provider_payload.get("reply_to_message_id") != 999 + + +@pytest.mark.asyncio +async def test_llm_and_category_results_keep_verified_callback_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway._security, "is_whitelisted", lambda _user_id: True) + + class _Redis: + async def get(self, _key: str) -> str: + return json.dumps({ + "name": "inspect", + "provider": "internal", + "tool": "status", + "risk": "low", + "incident_id": "INC-1", + }) + + monkeypatch.setattr(gateway_module, "get_redis", lambda: _Redis()) + monkeypatch.setattr( + callback_dispatcher, + "dispatch_llm_action", + lambda _action, _context: {"ok": True}, + ) + await gateway._handle_llm_action_callback( + callback_query_id="callback-1", + callback_data="la:short", + user_id=42, + username="operator", + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + llm_payload = next( + call.args[1] + for call in send_request.await_args_list + if call.args[0] == "sendMessage" + ) + _assert_verified_callback_payload(llm_payload) + + send_request.reset_mock() + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + monkeypatch.setattr( + callback_dispatcher, + "get_action_spec", + lambda _action: SimpleNamespace(emoji="🔎"), + ) + monkeypatch.setattr( + callback_dispatcher, + "dispatch_action", + AsyncMock(return_value=DispatchResult( + success=True, + action="inspect", + incident_id="INC-1", + user_id=42, + result_text="result", + )), + ) + + class _IncidentRepo: + async def get_by_id(self, _incident_id: str): + return SimpleNamespace(signals=[]) + + monkeypatch.setattr( + incident_repository, + "get_incident_repository", + lambda: _IncidentRepo(), + ) + await gateway._dispatch_category_action( + callback_query_id="callback-2", + action="inspect", + incident_id="INC-1", + user_id=42, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + _assert_verified_callback_payload(send_request.await_args.args[1]) + + +@pytest.mark.asyncio +async def test_write_category_callback_keeps_verified_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr( + gateway._security, + "parse_callback_data", + lambda _data: { + "action": "restart_safe", + "approval_id": "INC-1", + "is_info_action": False, + "nonce": "nonce", + }, + ) + monkeypatch.setattr( + gateway._security, + "verify_callback", + AsyncMock(return_value={"id": 42}), + ) + monkeypatch.setattr(gateway, "_check_incident_state_guard", AsyncMock(return_value=None)) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + spec = SimpleNamespace( + requires_multi_sig=False, + risk="medium", + mcp_provider="internal", + mcp_tool="restart_safe", + emoji="▶️", + label="restart", + ) + monkeypatch.setattr(callback_dispatcher, "get_action_spec", lambda _action: spec) + monkeypatch.setattr( + callback_dispatcher, + "dispatch_action", + AsyncMock(return_value=DispatchResult( + success=True, + action="restart_safe", + incident_id="INC-1", + user_id=42, + result_text="done", + )), + ) + + class _IncidentRepo: + async def get_by_id(self, _incident_id: str): + return SimpleNamespace(incident_id="INC-1", signals=[]) + + monkeypatch.setattr( + incident_repository, + "get_incident_repository", + lambda: _IncidentRepo(), + ) + + result = await gateway.handle_callback( + callback_query_id="callback-1", + callback_data="restart_safe:INC-1:nonce", + user_id=42, + message_id=_MESSAGE_ID, + username="operator", + chat_id=_CHAT_ID, + ) + + assert result["success"] is True + _assert_verified_callback_payload(send_request.await_args.args[1]) + + +@pytest.mark.asyncio +async def test_approval_replies_keep_verified_callback_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID, + incident_id="INC-1", + action="approve", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + status_payload = next( + call.args[1] + for call in send_request.await_args_list + if call.args[0] == "sendMessage" + ) + _assert_verified_callback_payload(status_payload) + source_extra = status_payload[gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] + merged = gateway_module._merge_outbound_source_envelope_extra({}, source_extra) + assert merged["callback_reply"]["parent_provider_message_id"] == str( + _MESSAGE_ID + ) + + send_request.reset_mock() + approval_id = UUID("44444444-4444-4444-4444-444444444444") + terminal = SimpleNamespace( + id=approval_id, + status=ApprovalStatus.EXECUTION_SUCCESS, + action="restart_safe", + metadata={}, + ) + + class _ApprovalService: + async def sign_approval(self, **_kwargs): + return terminal, "already done", False + + monkeypatch.setattr(approval_db, "get_approval_service", lambda: _ApprovalService()) + await gateway._execute_approval_action( + action="approve", + approval_id=str(approval_id), + user_id=42, + username="operator", + message_id=_MESSAGE_ID, + inbound_chat_id=_CHAT_ID, + ) + _assert_verified_callback_payload(send_request.await_args.args[1]) + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + ({"ok": True}, True), + ({"ok": True, "_awooop_delivery_status": "sent_reused"}, True), + ({"ok": False}, False), + ( + {"ok": True, "_awooop_delivery_status": "blocked_no_egress"}, + False, + ), + ({"ok": True, "_awooop_provider_send_performed": False}, False), + ( + { + "ok": True, + "_awoooi_canonical_route_receipt": { + "provider_send_performed": False, + }, + }, + False, + ), + ], +) +def test_telegram_send_delivery_requires_affirmative_provider_receipt( + result: dict, + expected: bool, +) -> None: + assert gateway_module._telegram_send_delivery_succeeded(result) is expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "primary_failure", + [ + {"ok": False}, + {"ok": True, "_awooop_delivery_status": "blocked_no_egress"}, + {"ok": True, "_awooop_provider_send_performed": False}, + { + "ok": True, + "_awoooi_canonical_route_receipt": { + "provider_send_performed": False, + }, + }, + ], +) +async def test_approval_structured_no_send_retries_plain_on_exact_parent( + monkeypatch: pytest.MonkeyPatch, + primary_failure: dict, +) -> None: + gateway = TelegramGateway() + attempts: list[tuple[str, dict]] = [] + send_results = iter([primary_failure, {"ok": True}]) + + async def structured_primary_failure(method: str, payload: dict) -> dict: + attempts.append((method, payload)) + if method == "sendMessage": + return next(send_results) + return {"ok": True} + + record_failure = AsyncMock(return_value=None) + shared_fallback = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", structured_primary_failure) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + monkeypatch.setattr(gateway, "send_alert_notification", shared_fallback) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID, + incident_id="INC-1", + action="approve", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_attempts = [payload for method, payload in attempts if method == "sendMessage"] + assert len(send_attempts) == 2 + for payload in send_attempts: + _assert_verified_callback_payload(payload) + assert send_attempts[0]["parse_mode"] == "HTML" + assert "parse_mode" not in send_attempts[1] + record_failure.assert_not_awaited() + shared_fallback.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_approval_structured_double_no_send_records_durable_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + attempts: list[tuple[str, dict]] = [] + send_results = iter([ + {"ok": True, "_awooop_delivery_status": "blocked_no_egress"}, + {"ok": True, "_awooop_provider_send_performed": False}, + ]) + + async def structured_no_send(method: str, payload: dict) -> dict: + attempts.append((method, payload)) + if method == "sendMessage": + return next(send_results) + return {"ok": True} + + record_failure = AsyncMock(return_value=None) + shared_fallback = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", structured_no_send) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + monkeypatch.setattr(gateway, "send_alert_notification", shared_fallback) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID, + incident_id="INC-1", + action="reject", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_attempts = [payload for method, payload in attempts if method == "sendMessage"] + assert len(send_attempts) == 2 + for payload in send_attempts: + _assert_verified_callback_payload(payload) + shared_fallback.assert_not_awaited() + record_failure.assert_awaited_once() + assert record_failure.await_args.kwargs["parent_message_id"] == _MESSAGE_ID + assert record_failure.await_args.kwargs["incident_id"] == "INC-1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "action", + ["tune", "snooze", "silence", "approve", "reject"], +) +async def test_callback_card_edits_use_only_verified_inbound_identity( + monkeypatch: pytest.MonkeyPatch, + action: str, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + + await gateway._update_message_after_action( + action=action, + username="operator", + original_text="original", + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + assert [call.args[0] for call in send_request.await_args_list] == [ + "editMessageReplyMarkup", + "editMessageText", + ] + for call in send_request.await_args_list: + payload = call.args[1] + assert payload["chat_id"] == _CHAT_ID + assert payload["message_id"] == _MESSAGE_ID + + +@pytest.mark.asyncio +async def test_resolved_state_guard_edits_only_verified_inbound_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + approval_id = UUID("55555555-5555-5555-5555-555555555555") + + class _ApprovalService: + async def get_approval_by_id(self, _approval_id: UUID): + return SimpleNamespace(incident_id="INC-1") + + class _IncidentRepo: + async def get_by_id(self, _incident_id: str): + return SimpleNamespace( + status=IncidentStatus.RESOLVED, + resolved_at=None, + ) + + monkeypatch.setattr(approval_db, "get_approval_service", lambda: _ApprovalService()) + monkeypatch.setattr( + incident_repository, + "get_incident_repository", + lambda: _IncidentRepo(), + ) + + result = await gateway._check_incident_state_guard( + approval_id=str(approval_id), + callback_query_id="callback-1", + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + original_text="original", + ) + + assert result == { + "blocked": True, + "reason": "already_resolved", + "approval_id": str(approval_id), + } + method, payload = send_request.await_args.args + assert method == "editMessageText" + assert payload["chat_id"] == _CHAT_ID + assert payload["message_id"] == _MESSAGE_ID + + +@pytest.mark.asyncio +async def test_approval_reply_failure_retries_exact_parent_then_records_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + attempts: list[tuple[str, dict]] = [] + + async def fail_replies(method: str, payload: dict) -> dict: + attempts.append((method, payload)) + if method == "sendMessage": + raise gateway_module.TelegramGatewayError("provider unavailable") + return {"ok": True} + + record_failure = AsyncMock(return_value=None) + shared_fallback = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", fail_replies) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + monkeypatch.setattr(gateway, "send_alert_notification", shared_fallback) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID, + incident_id="INC-1", + action="reject", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_attempts = [payload for method, payload in attempts if method == "sendMessage"] + assert len(send_attempts) == 2 + for payload in send_attempts: + _assert_verified_callback_payload(payload) + extra = payload[gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] + merged = gateway_module._merge_outbound_source_envelope_extra({}, extra) + assert merged["callback_reply"]["parent_provider_message_id"] == str( + _MESSAGE_ID + ) + assert "parse_mode" in send_attempts[0] + assert "parse_mode" not in send_attempts[1] + shared_fallback.assert_not_awaited() + record_failure.assert_awaited_once() + assert record_failure.await_args.kwargs["parent_message_id"] == _MESSAGE_ID + assert record_failure.await_args.kwargs["incident_id"] == "INC-1" + + +@pytest.mark.asyncio +async def test_approval_reply_parent_mismatch_is_durable_no_send_terminal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + record_failure = AsyncMock(return_value=None) + shared_fallback = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + monkeypatch.setattr(gateway, "send_alert_notification", shared_fallback) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID + 1, + incident_id="INC-1", + action="approve", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_request.assert_not_awaited() + shared_fallback.assert_not_awaited() + record_failure.assert_awaited_once() + assert record_failure.await_args.kwargs["parent_message_id"] == _MESSAGE_ID + + +class _LifecycleRedis: + async def get(self, key: str) -> str: + if key.startswith("tg_approval:") or key.startswith("tg_msg:"): + return str(_MESSAGE_ID) + raise AssertionError(key) + + async def set(self, *_args, **_kwargs) -> bool: + return True + + +@pytest.mark.asyncio +async def test_autonomous_lifecycle_threads_use_canonical_route_and_parent_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway_module, "get_redis", lambda: _LifecycleRedis()) + + assert await gateway.mark_auto_repaired("INC-1", "playbook", 10) is True + assert await gateway.append_incident_update("INC-2", "healthy") is True + assert await gateway.append_grouped_alert_digest( + incident_id="INC-3", + group_key="group-1", + digest_text="digest", + ) is True + + send_payloads = [ + call.args[1] + for call in send_request.await_args_list + if call.args[0] == "sendMessage" + ] + assert len(send_payloads) == 3 + parent_lookup_keys = [] + for payload in send_payloads: + assert gateway_module._NON_MONITORING_INTERACTION_KEY not in payload + assert payload[gateway_module._CANONICAL_ROUTE_CONTEXT_KEY] == { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + } + extra = payload[gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] + merged = gateway_module._merge_outbound_source_envelope_extra({}, extra) + receipt = merged["incident_lifecycle_parent_receipt"] + assert receipt["parent_provider_message_id"] == str(_MESSAGE_ID) + assert receipt["durable_mirror_required"] is True + parent_lookup_keys.append(receipt["parent_lookup_key"]) + assert merged["source_refs"]["parent_provider_message_ids"] == [ + str(_MESSAGE_ID) + ] + assert parent_lookup_keys == [ + "tg_approval:INC-1", + "tg_msg:INC-2", + "tg_msg:INC-3", + ] + + +@pytest.mark.asyncio +async def test_callback_fails_closed_before_dispatch_without_inbound_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + llm_handler = AsyncMock() + monkeypatch.setattr(gateway, "_handle_llm_action_callback", llm_handler) + + result = await gateway.handle_callback( + callback_query_id="callback-1", + callback_data="la:short", + user_id=42, + message_id=_MESSAGE_ID, + chat_id=None, + ) + + assert result["success"] is False + assert result["reason"] == "missing_verified_interaction_context" + llm_handler.assert_not_awaited() diff --git a/apps/api/tests/test_telegram_canonical_routing_registry.py b/apps/api/tests/test_telegram_canonical_routing_registry.py new file mode 100644 index 000000000..75f7e36c8 --- /dev/null +++ b/apps/api/tests/test_telegram_canonical_routing_registry.py @@ -0,0 +1,494 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from src.services.notification_matrix import ( + load_canonical_telegram_routing_registry, + resolve_canonical_telegram_route, +) + +EXPECTED_PRODUCTS = { + "2026fifa", + "agent-bounty-protocol", + "awooogo", + "awoooi", + "bitan-pharmacy", + "clawbot-openclaw", + "momo-pro", + "stockplatform-v2", + "tsenyang-website", + "vibework", + "vtuber", +} + + +def test_registry_covers_all_11_products_and_reconciles_rollups() -> None: + registry = load_canonical_telegram_routing_registry() + + assert { + product["product_id"] for product in registry["products"] + } == EXPECTED_PRODUCTS + assert registry["rollups"] == { + "product_count": 11, + "route_count": 20, + "allowed_route_count": 4, + "blocked_route_count": 13, + "disabled_route_count": 2, + "not_implemented_route_count": 1, + "shared_sre_allowed_route_count": 4, + "raw_numeric_destination_count": 0, + "runtime_route_change_count": 0, + "telegram_send_count": 0, + } + + +@pytest.mark.parametrize( + "signal_family", + [ + "shared_infrastructure", + "security_incident", + "disaster_recovery", + "incident_lifecycle", + ], +) +@pytest.mark.parametrize("severity", ["P0", "P1"]) +def test_shared_sre_accepts_only_allowlisted_p0_p1_lifecycle( + signal_family: str, severity: str +) -> None: + decision = resolve_canonical_telegram_route("awoooi", signal_family, severity) + + assert decision["decision"] == "allow" + assert decision["chat_alias"] == "awoooi_sre_war_room" + assert decision["allowed_destination"] == ( + "telegram_chat_alias:awoooi_sre_war_room" + ) + assert decision["receipt_backend"] == ( + "awoooi.alert_operation_log+telegram_outbound_receipts" + ) + + +@pytest.mark.parametrize( + ("signal_family", "severity"), + [ + ("product_raw_monitoring", "P2"), + ("product_raw_monitoring", "P3"), + ("marketing", "P1"), + ], +) +def test_shared_sre_blocks_raw_p2_p3_and_marketing( + signal_family: str, severity: str +) -> None: + decision = resolve_canonical_telegram_route("awoooi", signal_family, severity) + + assert decision["decision"] == "block" + assert decision["allowed_destination"] == "blocked" + + +@pytest.mark.parametrize("product_id", ["awooogo", "vibework", "vtuber"]) +def test_disabled_or_not_implemented_products_never_resolve_to_egress( + product_id: str, +) -> None: + decision = resolve_canonical_telegram_route(product_id, "any-signal", "P0") + + assert decision["decision"] == "block" + assert decision["egress_status"] in {"disabled", "not_implemented"} + assert decision["allowed_destination"] == "blocked" + + +@pytest.mark.parametrize( + ("product_id", "signal_family", "severity"), + [ + ("unknown-product", "shared_infrastructure", "P0"), + ("awoooi", "unknown-family", "P0"), + ("awoooi", "shared_infrastructure", "P3"), + ("awoooi", "shared_infrastructure", "SEV-1"), + ], +) +def test_unknown_route_dimensions_are_fail_closed( + product_id: str, signal_family: str, severity: str +) -> None: + decision = resolve_canonical_telegram_route(product_id, signal_family, severity) + + assert decision["decision"] == "block" + assert decision["route_id"] is None + assert decision["allowed_destination"] == "blocked" + assert decision["reason"] == "unknown_product_signal_or_severity" + + +def test_requested_destination_must_match_the_canonical_alias() -> None: + decision = resolve_canonical_telegram_route( + "awoooi", + "security_incident", + "P0", + requested_destination="telegram_chat_alias:some_other_room", + ) + + assert decision["decision"] == "block" + assert decision["reason"] == "requested_destination_not_allowed" + + +def test_explicit_empty_registry_is_invalid_instead_of_loading_default() -> None: + with pytest.raises(ValueError, match="schema_version"): + resolve_canonical_telegram_route( + "awoooi", + "security_incident", + "P0", + registry={}, + ) + + +def test_product_level_blocked_disposition_prevents_route_egress() -> None: + decision = resolve_canonical_telegram_route( + "agent-bounty-protocol", + "a2a_task_lifecycle", + "P1", + ) + + assert decision["decision"] == "block" + assert decision["egress_status"] == "blocked" + assert decision["allowed_destination"] == "blocked" + + +def test_registry_contains_only_symbolic_aliases_not_runtime_identifiers() -> None: + registry = load_canonical_telegram_routing_registry() + + serialized = json.dumps(registry, ensure_ascii=False) + assert '"chat_id"' not in serialized + assert '"bot_token"' not in serialized + for route in registry["routes"]: + for field in ("bot_alias", "chat_alias", "allowed_destination"): + assert not str(route[field]).lstrip("-").isdigit() + + +def test_shared_monitoring_routes_bind_tsenyang_owner_and_destination_policy() -> None: + registry = load_canonical_telegram_routing_registry() + shared_routes = [ + route + for route in registry["routes"] + if route["egress_status"] == "allowed" and route["scope"] == "shared" + ] + assert shared_routes + assert {route["bot_alias"] for route in shared_routes} == {"tsenyang_bot"} + + owner_role = next( + role + for role in registry["destination_roles"] + if role["chat_alias"] == "awoooi_sre_war_room" + and role["bot_alias"] == "tsenyang_bot" + ) + assert owner_role["bot_owner"] == "sre_shared_runtime" + assert owner_role["monitoring_owner"] is True + assert owner_role["monitoring_egress_policy"] == "allowlist_only" + + +@pytest.mark.parametrize( + ("field", "value", "error"), + [ + ("bot_owner", "ai_agent_runtime", "bot ownership"), + ("monitoring_owner", False, "not the monitoring owner"), + ( + "monitoring_egress_policy", + "interaction_reply_only_no_monitoring_ownership", + "not monitoring allowlisted", + ), + ], +) +def test_tampered_shared_destination_ownership_is_rejected( + tmp_path: Path, + field: str, + value: object, + error: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + role = next( + role + for role in registry["destination_roles"] + if role["chat_alias"] == "awoooi_sre_war_room" + and role["bot_alias"] == "tsenyang_bot" + ) + role[field] = value + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match=error): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_shared_route_cannot_switch_to_openclaw_bot(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route + for route in registry["routes"] + if route["route_id"] == "awoooi-security-incident" + ) + route["bot_alias"] = "openclaw_bot" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="shared SRE P0/P1"): + load_canonical_telegram_routing_registry(tmp_path) + + +@pytest.mark.parametrize("visible_label", ["TsenYang", "小O_小龍蝦"]) +def test_visible_labels_do_not_claim_proven_runtime_alias( + tmp_path: Path, + visible_label: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + role = next( + role + for role in registry["destination_roles"] + if role["visible_label"] == visible_label + ) + assert role["runtime_alias_evidence"] == "visible_label_only" + assert role["proven_runtime_alias"] == "none" + role["proven_runtime_alias"] = role["chat_alias"] + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="visible label must not claim"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_direct_egress_scanners_cover_agent99_and_reboot_recovery() -> None: + repo_root = Path(__file__).resolve().parents[3] + scanner_paths = [ + repo_root / "scripts/security/telegram-notification-egress-inventory.py", + repo_root + / "scripts/security/telegram-notification-egress-no-new-bypass-guard.py", + ] + for scanner_path in scanner_paths: + source = scanner_path.read_text(encoding="utf-8") + assert 'Path("agent99-control-plane.ps1")' in source + assert 'Path("scripts/reboot-recovery")' in source + assert '".ps1"' in source + + guarded_source = "\n".join( + [ + (repo_root / "agent99-control-plane.ps1").read_text(encoding="utf-8"), + *( + path.read_text(encoding="utf-8", errors="replace") + for path in sorted((repo_root / "scripts/reboot-recovery").rglob("*")) + if path.is_file() and path.suffix in {".ps1", ".sh", ".py"} + ), + ] + ) + assert "api.telegram.org/bot" not in guarded_source + + +def test_tampered_shared_sre_route_is_rejected(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route + for route in registry["routes"] + if route["route_id"] == "awoooi-shared-infrastructure" + ) + route["severity"].append("P3") + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="shared SRE P0/P1"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_allowed_route_without_receipt_is_rejected(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route for route in registry["routes"] if route["egress_status"] == "allowed" + ) + route["receipt_backend"] = "none" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="receipt backend"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_unknown_destination_is_rejected(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route for route in registry["routes"] if route["egress_status"] == "blocked" + ) + route["allowed_destination"] = "telegram_chat_alias:guessed_room" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="non-allowed route must be blocked"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_raw_chat_identifier_is_rejected(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + registry["routes"][0]["chat_alias"] = str(-(10**12)) + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="raw numeric runtime identifier"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_blocked_product_cannot_gain_an_allowed_route(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route + for route in registry["routes"] + if route["route_id"] == "bitan-product-health" + ) + route.update( + { + "bot_alias": "bitan_bot", + "chat_alias": "bitan_product_ops", + "allowed_destination": "telegram_chat_alias:bitan_product_ops", + "receipt_backend": "bitan.delivery_receipts", + "egress_status": "allowed", + } + ) + _write_registry(tmp_path, registry) + + with pytest.raises( + ValueError, match="bitan-pharmacy must not have an allowed route" + ): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_recursive_guard_rejects_sensitive_runtime_key(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + registry["destination_roles"][0]["bot_token"] = "redacted-is-still-forbidden" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="forbidden sensitive runtime key"): + load_canonical_telegram_routing_registry(tmp_path) + + +@pytest.mark.parametrize( + "sensitive_key", + [ + "api_key", + "password", + "credential", + "private_key", + "cookie", + "session", + ], +) +def test_recursive_guard_rejects_extended_sensitive_runtime_keys( + tmp_path: Path, + sensitive_key: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + registry["destination_roles"][0][sensitive_key] = "redacted" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="forbidden sensitive runtime key"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_recursive_guard_rejects_secret_shaped_runtime_value(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + registry["destination_roles"][0]["role"] = f"{10**7}:{'x' * 24}" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="secret-shaped runtime value"): + load_canonical_telegram_routing_registry(tmp_path) + + +@pytest.mark.parametrize( + ("section", "field"), + [ + ("top", "status"), + ("top", "destination_roles"), + ("product", "owner"), + ("route", "signal_family"), + ("policy", "default_decision"), + ("rollups", "route_count"), + ("destination_role", "monitoring_egress_policy"), + ("supersedes", "owner"), + ("execution_boundaries", "telegram_send_allowed"), + ], +) +def test_runtime_validator_rejects_missing_fields_at_every_object_level( + tmp_path: Path, + section: str, + field: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + target = _registry_object_for_section(registry, section) + target.pop(field) + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="missing required fields"): + load_canonical_telegram_routing_registry(tmp_path) + + +@pytest.mark.parametrize( + "section", + [ + "top", + "product", + "route", + "policy", + "rollups", + "destination_role", + "supersedes", + "execution_boundaries", + ], +) +def test_runtime_validator_rejects_unexpected_fields_at_every_object_level( + tmp_path: Path, + section: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + target = _registry_object_for_section(registry, section) + target["unexpected_field"] = "must-fail" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="unexpected fields"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_schema_closes_every_data_object_and_validates_snapshot() -> None: + import jsonschema + + repo_root = Path(__file__).resolve().parents[3] + schema = json.loads( + ( + repo_root + / "docs/schemas/telegram_canonical_routing_registry_v1.schema.json" + ).read_text(encoding="utf-8") + ) + + def assert_closed_objects(node: object, path: str = "schema") -> None: + if isinstance(node, dict): + if node.get("type") == "object": + assert node.get("additionalProperties") is False, path + for key, nested in node.items(): + assert_closed_objects(nested, f"{path}.{key}") + elif isinstance(node, list): + for index, nested in enumerate(node): + assert_closed_objects(nested, f"{path}[{index}]") + + assert_closed_objects(schema) + jsonschema.Draft202012Validator.check_schema(schema) + jsonschema.Draft202012Validator(schema).validate( + load_canonical_telegram_routing_registry() + ) + + +def _registry_object_for_section(registry: dict, section: str) -> dict: + if section == "top": + return registry + if section == "product": + return next( + product + for product in registry["products"] + if product["product_id"] == "awoooi" + ) + if section == "route": + return registry["routes"][0] + if section == "destination_role": + return registry["destination_roles"][0] + return registry[section] + + +def _write_registry(directory: Path, registry: dict) -> None: + (directory / "telegram-canonical-routing-registry.snapshot.json").write_text( + json.dumps(registry), + encoding="utf-8", + ) diff --git a/apps/api/tests/test_telegram_canonical_sender_gate.py b/apps/api/tests/test_telegram_canonical_sender_gate.py new file mode 100644 index 000000000..2231e277a --- /dev/null +++ b/apps/api/tests/test_telegram_canonical_sender_gate.py @@ -0,0 +1,521 @@ +from __future__ import annotations + +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +import src.db.base as db_base +import src.services.approval_db as approval_db +import src.services.telegram_gateway as gateway_module +from src.services.telegram_gateway import TelegramGateway + + +class _FakeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return {"ok": True, "result": {"message_id": 11}} + + +class _FakeHttpClient: + def __init__(self) -> None: + self.posts: list[tuple[str, dict]] = [] + + async def post(self, url: str, json: dict) -> _FakeResponse: + self.posts.append((url, json)) + return _FakeResponse() + + +def _prepared_gateway(monkeypatch: pytest.MonkeyPatch) -> tuple[TelegramGateway, _FakeHttpClient]: + gateway = TelegramGateway() + client = _FakeHttpClient() + gateway._initialized = True + gateway._http_client = client # type: ignore[assignment] + monkeypatch.setattr(gateway_module.settings, "SRE_GROUP_CHAT_ID", "test-room") + monkeypatch.setattr(gateway, "_attach_incident_thread_reply", AsyncMock()) + monkeypatch.setattr(gateway, "_mirror_outbound_message", AsyncMock()) + monkeypatch.setattr( + gateway._security, + "generate_callback_nonce", + lambda approval_id, action: f"{action}:test", + ) + return gateway, client + + +def _notification_message(): + from src.services.notifications.base import ExecutionStatus, NotificationMessage + + return NotificationMessage( + execution_status=ExecutionStatus.SUCCESS, + action_title="bounded test", + action_description="no provider egress", + approval_id="APR-TEST", + ) + + +@pytest.mark.asyncio +async def test_notification_provider_does_not_report_success_for_gateway_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services.notifications import telegram as provider_module + from src.services.notifications.base import NotificationStatus + + monkeypatch.setattr(provider_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(provider_module.settings, "SRE_GROUP_CHAT_ID", "test-room") + gateway = SimpleNamespace( + send_alert_notification=AsyncMock( + return_value={ + "ok": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + } + ) + ) + monkeypatch.setattr(gateway_module, "get_telegram_gateway", lambda: gateway) + + result = await provider_module.TelegramWebhookProvider().send( + _notification_message() + ) + + assert result.status is NotificationStatus.SKIPPED + assert result.message == "Telegram notification blocked before provider send" + assert result.response_data == { + "ok": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + } + gateway.send_alert_notification.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_notification_provider_requires_affirmative_delivery_ack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services.notifications import telegram as provider_module + from src.services.notifications.base import NotificationStatus + + monkeypatch.setattr(provider_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(provider_module.settings, "SRE_GROUP_CHAT_ID", "test-room") + gateway = SimpleNamespace( + send_alert_notification=AsyncMock( + return_value={"ok": False, "description": "provider rejected"} + ) + ) + monkeypatch.setattr(gateway_module, "get_telegram_gateway", lambda: gateway) + + result = await provider_module.TelegramWebhookProvider().send( + _notification_message() + ) + + assert result.status is NotificationStatus.FAILED + assert result.message == "Telegram provider did not acknowledge delivery" + assert result.error == "delivery_not_acknowledged" + + +@pytest.mark.asyncio +async def test_unclassified_send_message_is_blocked_before_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway._send_request( + "sendMessage", + {"chat_id": "test-room", "text": "unclassified"}, + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awooop_provider_send_performed"] is False + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_info_and_business_methods_are_blocked_before_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + info = await gateway.send_info_notification( + incident_id="INC-TEST", + title="FYI", + message="healthy", + ) + business = await gateway.send_business_alert( + incident_id="INC-TEST", + alertname="BusinessMetric", + business_domain="test", + metric_name="rate", + current_value="1", + threshold="2", + ) + + assert info["_awooop_delivery_status"] == "blocked_no_egress" + assert business["_awooop_delivery_status"] == "blocked_no_egress" + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_wrong_group_override_is_blocked_before_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway.send_escalation_card( + incident_id="INC-TEST", + original_alertname="Outage", + duration_min=15, + group_chat_id="wrong-room", + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awoooi_canonical_route_receipt"]["classifier"] == ( + "caller_destination_mismatch" + ) + assert client.posts == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method_name", ["meta", "secops", "escalation"]) +async def test_allowed_legacy_monitoring_methods_use_canonical_provider_once( + monkeypatch: pytest.MonkeyPatch, + method_name: str, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + if method_name == "meta": + result = await gateway.send_meta_alert( + incident_id="INC-TEST", + approval_id="APR-TEST", + alertname="AlertChainBroken", + alert_category="alertchain_health", + diagnosis="test", + ) + elif method_name == "secops": + result = await gateway.send_secops_card( + incident_id="INC-TEST", + approval_id="APR-TEST", + alertname="Threat", + threat_level="critical", + ) + else: + result = await gateway.send_escalation_card( + incident_id="INC-TEST", + original_alertname="Outage", + duration_min=15, + ) + + assert result["ok"] is True + assert result["_awoooi_canonical_route_receipt"]["decision"] == "allowed" + assert result["_awoooi_canonical_route_receipt"]["provider_send_performed"] is True + assert len(client.posts) == 1 + assert client.posts[0][1]["chat_id"] == "test-room" + + +@pytest.mark.asyncio +async def test_openclaw_cannot_own_tsenyang_monitoring_route( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway._send_as_bot( + token="test-token-placeholder", + text="must not send", + product_id="awoooi", + signal_family="security_incident", + severity="P1", + actual_bot_alias="openclaw_bot", + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awoooi_canonical_route_receipt"]["classifier"] == ( + "sender_bot_alias_not_authorized_for_route" + ) + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_operator_reply_lane_and_runtime_edit_binding_remain_available( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + assert gateway.chat_id == "test-room" + assert gateway.alert_chat_id == "test-room" + unverified = await gateway.send_notification( + "must not infer an operator reply", + chat_id="operator-room", + ) + + assert unverified["_awooop_delivery_status"] == "blocked_no_egress" + assert unverified["_awoooi_canonical_route_receipt"]["classifier"] == ( + "missing_product_signal_severity_or_interaction_context" + ) + assert client.posts == [] + + result = await gateway.send_notification( + "command reply", + chat_id="operator-room", + interaction_kind="operator_command_reply", + inbound_chat_id="operator-room", + inbound_message_id=42, + ) + + assert result["ok"] is True + receipt = result["_awoooi_canonical_route_receipt"] + assert receipt["classifier"] == "non_monitoring_interaction_allowed" + assert receipt["destination_alias"] == "inbound_interaction_reply_target" + assert client.posts[0][1]["chat_id"] == "operator-room" + assert client.posts[0][1]["reply_to_message_id"] == 42 + + +@pytest.mark.asyncio +async def test_interaction_reply_must_match_verified_inbound_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway._send_request( + "sendMessage", + { + "chat_id": "operator-room", + "text": "mismatched reply", + "reply_to_message_id": 43, + gateway_module._NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "operator_command_reply", + "inbound_chat_id": "operator-room", + "inbound_message_id": 42, + }, + }, + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awoooi_canonical_route_receipt"]["classifier"] == ( + "interaction_reply_context_missing_or_mismatched" + ) + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_t1_through_t6_use_only_canonical_monitoring_routes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + canonical_send = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "send_canonical_message", canonical_send) + + await gateway.send_guardrail_blocked("api", "HighCpu", "protected") + await gateway.send_preflight_failed("api", 25.0, 24.0, "backup-1") + await gateway.send_backup_result("backup-1", True) + await gateway.send_multisig_waiting("repair", "api", 1, 2, "APR-1") + await gateway.send_multisig_approved("repair", "api") + await gateway.send_change_applied("operator", "repair", "2026-07-14T00:00:00Z") + + assert canonical_send.await_count == 6 + assert [ + ( + call.kwargs["product_id"], + call.kwargs["signal_family"], + call.kwargs["severity"], + ) + for call in canonical_send.await_args_list + ] == [ + ("awoooi", "security_incident", "P1"), + ("awoooi", "disaster_recovery", "P1"), + ("awoooi", "disaster_recovery", "P1"), + ("awoooi", "incident_lifecycle", "P1"), + ("awoooi", "incident_lifecycle", "P1"), + ("awoooi", "incident_lifecycle", "P1"), + ] + + +@pytest.mark.asyncio +async def test_image_analysis_reply_preserves_original_inbound_message_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services.image_analysis_service import ImageAnalysisService + + gateway = SimpleNamespace( + initialize=AsyncMock(), + send_as_openclaw=AsyncMock(return_value={"ok": True}), + ) + service = ImageAnalysisService() + monkeypatch.setattr( + gateway_module, + "get_telegram_gateway", + lambda: gateway, + ) + monkeypatch.setattr( + service, + "_download_telegram_file", + AsyncMock(return_value=b"image"), + ) + monkeypatch.setattr( + service, + "analyze", + AsyncMock(return_value="analysis"), + ) + + await service.download_and_analyze( + chat_id="test-room", + file_id="file-1", + original_message_id=77, + ) + + gateway.initialize.assert_awaited_once() + gateway.send_as_openclaw.assert_awaited_once_with( + "🖼️ 圖片分析\nanalysis", + reply_to_message_id=77, + inbound_chat_id="test-room", + ) + + +@pytest.mark.asyncio +async def test_manual_repair_callback_replies_to_verified_original_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + monkeypatch.setattr( + gateway._security, + "parse_callback_data", + lambda _data: { + "action": "log_manual_fix", + "approval_id": "APR-1", + "is_info_action": False, + "nonce": "nonce-1", + }, + ) + monkeypatch.setattr( + gateway._security, + "verify_callback", + AsyncMock(return_value={"id": 123, "username": "operator"}), + ) + monkeypatch.setattr( + gateway, + "_check_incident_state_guard", + AsyncMock(return_value=None), + ) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + monkeypatch.setattr(gateway_module, "get_redis", lambda: _FakeRedis()) + + result = await gateway.handle_callback( + callback_query_id="callback-1", + callback_data="log_manual_fix:APR-1:nonce-1", + user_id=123, + message_id=77, + chat_id="operator-room", + username="operator", + ) + + assert result["success"] is True + assert result["waiting_for_manual_fix"] is True + assert len(client.posts) == 1 + payload = client.posts[0][1] + assert payload["chat_id"] == "operator-room" + assert payload["reply_to_message_id"] == 77 + + +@pytest.mark.asyncio +async def test_manual_repair_flow_blocks_before_side_effect_without_inbound_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway.handle_manual_fix_done( + user_id=123, + username="operator", + fix_steps="restart", + ) + + assert result == { + "success": False, + "reason": "missing_verified_interaction_context", + } + assert client.posts == [] + + +class _FakeDb: + def __init__(self) -> None: + self.params: list[dict] = [] + + async def execute(self, statement: object, params: dict) -> None: + del statement + self.params.append(params) + + +class _FakeRedis: + async def setex(self, key: str, ttl: int, value: str) -> None: + del key, ttl, value + + +def _patch_approval_dependencies( + monkeypatch: pytest.MonkeyPatch, + gateway: TelegramGateway, +) -> _FakeDb: + fake_db = _FakeDb() + + @asynccontextmanager + async def fake_db_context(*args, **kwargs): + del args, kwargs + yield fake_db + + monkeypatch.setattr(db_base, "get_db_context", fake_db_context) + monkeypatch.setattr( + approval_db, + "get_approval_service", + lambda: SimpleNamespace(update_telegram_message=True), + ) + monkeypatch.setattr(gateway_module, "get_redis", lambda: _FakeRedis()) + monkeypatch.setattr( + gateway_module, + "_fetch_remediation_summary_for_card", + AsyncMock(return_value=None), + ) + monkeypatch.setattr(gateway, "_build_inline_keyboard", AsyncMock(return_value={})) + return fake_db + + +@pytest.mark.asyncio +async def test_approval_missing_type_is_blocked_without_provider_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + _patch_approval_dependencies(monkeypatch, gateway) + + result = await gateway.send_approval_card( + approval_id="APR-TEST", + risk_level="critical", + resource_name="resource", + root_cause="cause", + suggested_action="review", + notification_type="", + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_allowed_type3_approval_persists_runtime_chat_without_public_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + fake_db = _patch_approval_dependencies(monkeypatch, gateway) + monkeypatch.setattr(gateway, "canonical_destination_chat_id", lambda **kwargs: "7") + + result = await gateway.send_approval_card( + approval_id="APR-TEST", + risk_level="critical", + resource_name="resource", + root_cause="cause", + suggested_action="review", + notification_type="TYPE-3", + ) + + assert result["ok"] is True + assert len(client.posts) == 1 + assert any(params.get("cid") == 7 for params in fake_db.params) + receipt = result["_awoooi_canonical_route_receipt"] + assert receipt["destination_alias"] == "awoooi_sre_war_room" + assert "chat_id" not in receipt diff --git a/apps/api/tests/test_telegram_delivery_truth_callers.py b/apps/api/tests/test_telegram_delivery_truth_callers.py new file mode 100644 index 000000000..604c6d904 --- /dev/null +++ b/apps/api/tests/test_telegram_delivery_truth_callers.py @@ -0,0 +1,399 @@ +"""Regression tests for callers that must distinguish routing from delivery.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from src.models.incident import Incident, Severity, Signal +from src.repositories.alert_operation_log_repository import ALERT_EVENT_TYPES +from src.services import decision_manager as decision_module +from src.services import report_generation_service as report_module +from src.services.ai_rate_limiter import AIRateLimiter +from src.services.approval_execution import ApprovalExecutionService +from src.services.report_generation_service import ReportGenerationService + +NO_SEND_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": { + "decision": "blocked", + "provider_send_performed": False, + }, +} + +SENT_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 42, "chat": {"id": -1001}}, +} + + +class FakeRedis: + def __init__(self) -> None: + self.values: dict[str, str] = {} + self.set_calls: list[tuple[str, str, int | None]] = [] + self.setex_calls: list[tuple[str, int, str]] = [] + + async def get(self, key: str): + return self.values.get(key) + + async def exists(self, key: str) -> bool: + return key in self.values + + async def set(self, key: str, value: str, ex: int | None = None, **_kwargs): + self.values[key] = value + self.set_calls.append((key, value, ex)) + return True + + async def setex(self, key: str, ttl: int, value: str) -> bool: + self.values[key] = value + self.setex_calls.append((key, ttl, value)) + return True + + +class FakeLogger: + def __init__(self) -> None: + self.events: list[tuple[str, str]] = [] + + def debug(self, event: str, **_kwargs) -> None: + self.events.append(("debug", event)) + + def info(self, event: str, **_kwargs) -> None: + self.events.append(("info", event)) + + def warning(self, event: str, **_kwargs) -> None: + self.events.append(("warning", event)) + + def error(self, event: str, **_kwargs) -> None: + self.events.append(("error", event)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method_name", ["_send_cost_alert", "_send_cost_warning"]) +@pytest.mark.parametrize("delivery_result", [NO_SEND_RECEIPT, None]) +async def test_cost_alert_dedup_is_written_only_after_affirmative_delivery( + monkeypatch, + method_name: str, + delivery_result: object, +) -> None: + redis = FakeRedis() + + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "sre-room" + + async def send_canonical_message(self, **_payload): + return delivery_result + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + limiter = AIRateLimiter() + limiter._redis = redis + + if method_name == "_send_cost_alert": + await limiter._send_cost_alert("gemini", 5.1, 5.0) + else: + await limiter._send_cost_warning("gemini", 4.1, 4.0) + + assert redis.set_calls == [] + + +@pytest.mark.asyncio +async def test_cost_alert_sets_dedup_after_affirmative_delivery(monkeypatch) -> None: + redis = FakeRedis() + + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "sre-room" + + async def send_canonical_message(self, **_payload): + return dict(SENT_RECEIPT) + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + limiter = AIRateLimiter() + limiter._redis = redis + + await limiter._send_cost_alert("gemini", 5.1, 5.0) + + assert len(redis.set_calls) == 1 + assert redis.set_calls[0][0] == "ai_rate:cost_alert_sent:gemini" + + +@pytest.mark.asyncio +async def test_p3_reports_fail_closed_before_collection_when_route_missing( + monkeypatch, +) -> None: + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "" + + async def send_to_group(self, *_args, **_kwargs): + raise AssertionError("provider must not be called without a route") + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + service = ReportGenerationService() + daily = AsyncMock() + monthly = AsyncMock() + monkeypatch.setattr(service, "collect_daily_kpi", daily) + monkeypatch.setattr(service, "collect_report_source_health", monthly) + + await service.send_daily_report() + monthly_result = await service.send_monthly_report() + + daily.assert_not_awaited() + monthly.assert_not_awaited() + assert monthly_result is False + + +@pytest.mark.asyncio +async def test_postmortem_no_ack_retries_and_verifies_fallback(monkeypatch) -> None: + send = AsyncMock(return_value=dict(NO_SEND_RECEIPT)) + + class Gateway: + send_to_group = send + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + monkeypatch.setattr(report_module.asyncio, "sleep", AsyncMock()) + service = ReportGenerationService() + monkeypatch.setattr(service, "_persist_postmortem_km", AsyncMock()) + now = datetime.now(UTC) + + await service.trigger_postmortem( + incident_id="INC-NO-ACK", + title="provider did not acknowledge", + created_at=now - timedelta(minutes=12), + resolved_at=now, + ) + + # Three bounded attempts plus one simplified fallback; all remain no-send. + assert send.await_count == 4 + + +@pytest.mark.asyncio +async def test_approval_blocked_receipt_writes_no_send_not_sent(monkeypatch) -> None: + redis = FakeRedis() + operations: list[dict] = [] + + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "sre-room" + + async def _send_request(self, *_args, **_kwargs): + return dict(SENT_RECEIPT) + + async def send_canonical_message(self, **_payload): + return dict(NO_SEND_RECEIPT) + + class OperationRepository: + async def append(self, event_type: str, **kwargs) -> None: + operations.append({"event_type": event_type, **kwargs}) + + class BrokenDB: + async def execute(self, *_args, **_kwargs): + raise RuntimeError("expected test DB bypass") + + @asynccontextmanager + async def fake_db_context(): + yield BrokenDB() + + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + monkeypatch.setattr("src.db.base.get_db_context", fake_db_context) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + monkeypatch.setattr( + "src.repositories.alert_operation_log_repository.get_alert_operation_log_repository", + lambda: OperationRepository(), + ) + monkeypatch.setattr( + "src.services.approval_execution.incident_truth_chain_button_row", + lambda _incident_id: None, + ) + + service = ApprovalExecutionService() + monkeypatch.setattr( + service, + "_fetch_operator_outcome_for_result", + AsyncMock( + return_value={ + "state": "completed_verified", + "summary_zh": "verified", + "needs_human": False, + "next_action": "monitor", + "notification": {"channels": []}, + "execution_result": {}, + } + ), + ) + approval = SimpleNamespace( + id="approval-no-ack", + incident_id="INC-NO-ACK", + action="kubectl rollout restart deployment/api", + requested_by="operator", + ) + + await service._push_execution_result_to_alert( + approval, + success=True, + error=None, + execution_kind="controlled_apply", + repair_executed=True, + repair_attempted=True, + ) + + assert [row["event_type"] for row in operations] == [ + "NOTIFICATION_CLASSIFIED" + ] + assert operations[0]["event_type"] in ALERT_EVENT_TYPES + assert operations[0]["action_detail"] == "telegram_execution_result_no_send" + assert operations[0]["success"] is False + assert operations[0]["context"]["execution_success"] is True + + +def _incident() -> Incident: + return Incident( + incident_id="INC-DECISION-NO-ACK", + severity=Severity.P3, + signals=[ + Signal( + alert_name="ContainerPressure", + severity=Severity.P3, + source="alertmanager", + fired_at=datetime.now(UTC), + labels={"alertname": "ContainerPressure"}, + ) + ], + affected_services=["api"], + ) + + +@pytest.mark.asyncio +async def test_decision_no_ack_does_not_set_fingerprint_dedup(monkeypatch) -> None: + from src.services.telegram_gateway import NotificationType + + redis = FakeRedis() + + class Gateway: + async def send_info_notification(self, **_payload): + return dict(NO_SEND_RECEIPT) + + class ApprovalService: + async def update_action_by_incident_id(self, **_payload) -> None: + return None + + monkeypatch.setattr(decision_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + monkeypatch.setattr( + "src.services.telegram_gateway.classify_notification", + lambda **_kwargs: NotificationType.TYPE_1, + ) + monkeypatch.setattr( + "src.services.approval_db.get_approval_service", + lambda: ApprovalService(), + ) + + await decision_module._push_decision_to_telegram( + _incident(), + { + "action": "OBSERVE", + "description": "bounded observation", + "reasoning": "診斷：bounded observation", + "risk_level": "low", + "confidence": 0.9, + "source": "test", + }, + ) + + assert not any(key.startswith("telegram_sent:fp:") for key in redis.values) + + +@pytest.mark.asyncio +async def test_log_summary_route_preflight_runs_before_llm(monkeypatch) -> None: + summarize = AsyncMock(return_value="summary") + + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "" + + async def send_as_nemotron(self, *_args, **_kwargs): + raise AssertionError("provider must not be called without a route") + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + monkeypatch.setattr( + "src.services.log_summary_service.get_log_summary_service", + lambda: SimpleNamespace(summarize_with_soft_timeout=summarize), + ) + + await decision_module._send_log_summary(_incident()) + + summarize.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_auto_repair_fallback_requires_affirmative_delivery(monkeypatch) -> None: + test_logger = FakeLogger() + + class Gateway: + async def append_incident_update(self, **_payload) -> bool: + return False + + def canonical_destination_chat_id(self, **_route) -> str: + return "sre-room" + + async def send_alert_notification(self, *_args, **_kwargs): + return dict(NO_SEND_RECEIPT) + + class DB: + async def execute(self, *_args, **_kwargs) -> None: + return None + + async def commit(self) -> None: + return None + + @asynccontextmanager + async def fake_db_context(): + yield DB() + + monkeypatch.setattr(decision_module, "logger", test_logger) + monkeypatch.setattr("src.db.base.get_db_context", fake_db_context) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + + await decision_module._push_auto_repair_result( + _incident(), + action="bounded observe", + success=False, + error="verifier_failed", + ) + + event_names = [event for _level, event in test_logger.events] + assert "auto_repair_result_no_send" in event_names + assert "auto_repair_result_sent" not in event_names diff --git a/apps/api/tests/test_telegram_delivery_truth_callers_v2.py b/apps/api/tests/test_telegram_delivery_truth_callers_v2.py new file mode 100644 index 000000000..841305d6b --- /dev/null +++ b/apps/api/tests/test_telegram_delivery_truth_callers_v2.py @@ -0,0 +1,394 @@ +"""Delivery-truth regressions for migrated Telegram callers. + +2026-07-14 Codex v1.0 (Asia/Taipei): an adapter may return ``ok=true`` while +the canonical egress gate reports ``provider_send_performed=false``. These +tests keep that structured no-send receipt from becoming sent logs, counters, +or durable dedup state. No Telegram/provider network is used. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from src.jobs import ai_slo_watchdog_job as watchdog_module +from src.jobs import capacity_forecaster_job as capacity_module +from src.jobs import hermes_rule_quality_job as hermes_module +from src.services import converged_alert_recurrence_notifier as recurrence_module +from src.services import decision_manager as decision_module +from src.services import failover_alerter as failover_module +from src.services import k3s_monitor_service as k3s_module +from src.services import weekly_report_service as weekly_module + +NO_PROVIDER_SEND_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": { + "decision": "blocked", + "provider_send_performed": False, + }, +} + +SENT_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 42, "chat": {"id": -1001}}, +} + + +class _Redis: + def __init__(self) -> None: + self.values: dict[str, str] = {} + self.deleted: list[str] = [] + + async def set( + self, + key: str, + value: str, + *, + ex: int | None = None, + nx: bool | None = None, + **_kwargs, + ) -> bool | None: + del ex + if nx and key in self.values: + return None + self.values[key] = value + return True + + async def delete(self, key: str) -> int: + self.deleted.append(key) + return 1 if self.values.pop(key, None) is not None else 0 + + async def exists(self, key: str) -> bool: + return key in self.values + + async def get(self, key: str) -> str | None: + return self.values.get(key) + + async def scan(self, _cursor: int, *, match: str, count: int): + del match, count + return 0, list(self.values) + + +class _Logger: + def __init__(self) -> None: + self.events: list[tuple[str, str]] = [] + + def debug(self, event: str, **_kwargs) -> None: + self.events.append(("debug", event)) + + def info(self, event: str, **_kwargs) -> None: + self.events.append(("info", event)) + + def warning(self, event: str, **_kwargs) -> None: + self.events.append(("warning", event)) + + def error(self, event: str, **_kwargs) -> None: + self.events.append(("error", event)) + + def exception(self, event: str, **_kwargs) -> None: + self.events.append(("exception", event)) + + +class _Gateway: + def __init__(self, receipt: dict) -> None: + self.receipt = receipt + + def canonical_destination_chat_id(self, **_route) -> str: + return "canonical-room" + + async def send_alert_notification(self, *_args, **_kwargs) -> dict: + return dict(self.receipt) + + async def send_meta_alert(self, **_kwargs) -> dict: + return dict(self.receipt) + + async def send_canonical_message(self, **_kwargs) -> dict: + return dict(self.receipt) + + async def send_text(self, *_args, **_kwargs) -> dict: + return dict(self.receipt) + + +@pytest.mark.asyncio +async def test_failover_commits_dedup_and_sent_log_only_after_provider_ack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _Redis() + logger = _Logger() + gateway = _Gateway(NO_PROVIDER_SEND_RECEIPT) + monkeypatch.setattr(failover_module, "logger", logger) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: gateway, + ) + alerter = failover_module.FailoverAlerter(redis_client=redis) + + await alerter.alert_failover({"to_provider": "ollama_gcp_b"}) + + dedup_key = "alert:failover:ollama_gcp_b:dedup" + assert dedup_key not in redis.values + assert ("info", "failover_alert_sent") not in logger.events + assert ("warning", "failover_alert_no_send") in logger.events + + gateway.receipt = SENT_RECEIPT + await alerter.alert_failover({"to_provider": "ollama_gcp_b"}) + + assert redis.values[dedup_key] == "sent" + assert ("info", "failover_alert_sent") in logger.events + + +@pytest.mark.asyncio +async def test_watchdog_releases_dedup_and_does_not_log_sent_without_provider_ack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import ai_slo_calculator, flywheel_stats_service, governance_agent + + class _SloCalculator: + async def calculate(self): + return SimpleNamespace(any_violated=False, metrics=[]) + + class _FlywheelStats: + async def compute(self): + return SimpleNamespace(execution_success_rate=1.0) + + class _GovernanceAgent: + async def check_trust_drift(self, *, emit_alert: bool): + assert emit_alert is False + return {"drifted": 0, "drift_ratio": 0.0} + + redis = _Redis() + logger = _Logger() + monkeypatch.setattr(watchdog_module, "logger", logger) + monkeypatch.setattr(watchdog_module, "get_redis", lambda: redis) + monkeypatch.setattr(watchdog_module, "_is_grace_active", AsyncMock(return_value=True)) + monkeypatch.setattr( + watchdog_module, + "_count_pending_no_tg_sent", + AsyncMock(return_value=watchdog_module._TG_SILENCE_THRESHOLD), + ) + monkeypatch.setattr( + watchdog_module, + "_count_approved_playbooks", + AsyncMock(return_value=(1, 1)), + ) + monkeypatch.setattr( + watchdog_module, + "_count_pending_stuck_analysis", + AsyncMock(return_value=0), + ) + monkeypatch.setattr(ai_slo_calculator, "AiSloCalculator", _SloCalculator) + monkeypatch.setattr(flywheel_stats_service, "FlywheelStatsService", _FlywheelStats) + monkeypatch.setattr( + governance_agent, + "get_governance_agent", + lambda: _GovernanceAgent(), + ) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT), + ) + + await watchdog_module._check_once() + + assert redis.values == {} + assert ("warning", "ai_slo_watchdog_alert_sent") not in logger.events + assert ("error", "ai_slo_watchdog_alert_no_send") in logger.events + + +@pytest.mark.asyncio +async def test_advisory_callers_reject_ok_true_without_provider_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import ai_advisory_helpers + + monkeypatch.setattr( + ai_advisory_helpers, + "is_snoozed", + AsyncMock(return_value=False), + ) + monkeypatch.setattr( + ai_advisory_helpers, + "build_ai_advisory_keyboard", + lambda **_kwargs: {}, + ) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT), + ) + + hermes_sent = await hermes_module._send_telegram_summary( + [ + { + "rule_name": "HighNoiseRule", + "noise_rate": 0.9, + "tp": 1, + "fp": 9, + } + ], + ) + capacity_sent = await capacity_module._send_telegram_forecast( + { + "host-a": [ + { + "reason": "disk trend", + "value": 0.9, + } + ] + }, + ) + + assert hermes_sent is False + assert capacity_sent is False + + +@pytest.mark.asyncio +async def test_report_callers_reject_ok_true_without_provider_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = _Gateway(NO_PROVIDER_SEND_RECEIPT) + monkeypatch.setattr(weekly_module, "get_telegram_gateway", lambda: gateway) + monkeypatch.setattr(k3s_module, "get_telegram_gateway", lambda: gateway) + + weekly = weekly_module.WeeklyReportService() + monkeypatch.setattr( + weekly, + "generate_report", + AsyncMock( + return_value=SimpleNamespace( + format=lambda: "weekly", + week_range="2026-W29", + ) + ), + ) + k3s = k3s_module.K3sMonitorService() + monkeypatch.setattr( + k3s, + "collect_cluster_status", + AsyncMock( + return_value=SimpleNamespace( + format=lambda: "daily", + report_date="2026-07-14", + ) + ), + ) + + assert await weekly.send_weekly_report() is False + assert await k3s.send_daily_report() is False + + +@pytest.mark.asyncio +async def test_recurrence_releases_reservation_and_never_logs_sent_without_ack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _Redis() + logger = _Logger() + monkeypatch.setattr(recurrence_module, "logger", logger) + monkeypatch.setattr(recurrence_module, "get_redis", lambda: redis) + monkeypatch.setattr( + recurrence_module, + "should_notify_converged_alert_recurrence", + AsyncMock(return_value=True), + ) + monkeypatch.setattr( + recurrence_module, + "get_telegram_gateway", + lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT), + ) + + await recurrence_module.notify_converged_alert_recurrence( + source="alertmanager", + fingerprint="fp-no-provider-send", + alertname="ServiceDown", + severity="critical", + namespace="awoooi-prod", + target_resource="api", + hit_count=2, + incident_id="INC-NO-PROVIDER-SEND", + approval_id="approval-1", + ) + + key = recurrence_module.converged_alert_recurrence_key("fp-no-provider-send") + assert key in redis.deleted + assert ("info", "converged_alert_recurrence_sent") not in logger.events + assert ("error", "converged_alert_recurrence_failed") in logger.events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider_ack, expected_resent", [(False, 0), (True, 1)]) +async def test_stale_token_replay_counts_only_awaited_delivery_receipts( + monkeypatch: pytest.MonkeyPatch, + provider_ack: bool, + expected_resent: int, +) -> None: + from src.db import base as db_base + from src.repositories import incident_repository + + incident_id = "INC-STALE-DELIVERY-TRUTH" + redis = _Redis() + redis.values["decision:DEC-STALE"] = json.dumps( + { + "state": decision_module.DecisionState.READY.value, + "incident_id": incident_id, + "proposal_data": {"description": "bounded replay"}, + "token": "DEC-STALE", + } + ) + incident = SimpleNamespace( + incident_id=incident_id, + status="open", + affected_services=["api"], + created_at=datetime.now(UTC), + signals=[ + SimpleNamespace( + labels={"alertname": "ServiceDown"}, + alert_name="ServiceDown", + annotations={}, + ) + ], + ) + + @asynccontextmanager + async def _db_context(): + yield object() + + class _IncidentRepository: + def __init__(self, _db) -> None: + pass + + async def get_by_id(self, requested_id: str): + assert requested_id == incident_id + return incident + + push = AsyncMock(return_value=provider_ack) + logger = _Logger() + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + monkeypatch.setattr(db_base, "get_db_context", _db_context) + monkeypatch.setattr( + incident_repository, + "IncidentDBRepository", + _IncidentRepository, + ) + monkeypatch.setattr(decision_module, "_push_decision_to_telegram", push) + monkeypatch.setattr(decision_module, "logger", logger) + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + + manager = object.__new__(decision_module.DecisionManager) + resent = await manager.resend_stale_ready_tokens() + + assert resent == expected_resent + push.assert_awaited_once() + if provider_ack: + assert ("info", "stale_ready_token_resent") in logger.events + else: + assert ("info", "stale_ready_token_resent") not in logger.events + assert ("warning", "stale_ready_token_no_send") in logger.events diff --git a/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py new file mode 100644 index 000000000..82d5fd041 --- /dev/null +++ b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py @@ -0,0 +1,460 @@ +"""Delivery-truth regressions for gateway and webhook Telegram callers.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from src.api.v1 import sentry_webhook, signoz_webhook, telegram, webhooks +from src.services import telegram_gateway as gateway_module +from src.services.telegram_gateway import ( + TelegramGateway, + _telegram_send_delivery_succeeded, +) + +NO_SEND_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": { + "decision": "blocked_no_egress", + "provider_send_performed": False, + }, +} + +SENT_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 42}, +} + + +class FakeLogger: + def __init__(self) -> None: + self.events: list[tuple[str, str]] = [] + + def debug(self, event: str, **_kwargs) -> None: + self.events.append(("debug", event)) + + def info(self, event: str, **_kwargs) -> None: + self.events.append(("info", event)) + + def warning(self, event: str, **_kwargs) -> None: + self.events.append(("warning", event)) + + def error(self, event: str, **_kwargs) -> None: + self.events.append(("error", event)) + + def exception(self, event: str, **_kwargs) -> None: + self.events.append(("exception", event)) + + +class FakeRedis: + def __init__(self, values: dict[str, str] | None = None) -> None: + self.values = dict(values or {}) + self.set_calls: list[tuple[str, str, dict]] = [] + + async def get(self, key: str): + return self.values.get(key) + + async def set(self, key: str, value: str, **kwargs): + self.set_calls.append((key, value, kwargs)) + if kwargs.get("nx") and key in self.values: + return False + self.values[key] = value + return True + + async def setex(self, key: str, _ttl: int, value: str): + self.values[key] = value + return True + + +def test_delivery_helper_rejects_truthy_structured_no_send() -> None: + assert _telegram_send_delivery_succeeded(dict(NO_SEND_RECEIPT)) is False + assert _telegram_send_delivery_succeeded({"ok": True}) is True + + +@pytest.mark.asyncio +async def test_mark_auto_repaired_returns_false_for_structured_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = FakeRedis({"tg_approval:APR-1": "42"}) + gateway = TelegramGateway() + + async def send_request(method: str, _payload: dict): + if method == "sendMessage": + return dict(NO_SEND_RECEIPT) + return {"ok": True} + + monkeypatch.setattr(gateway_module, "get_redis", lambda: redis) + monkeypatch.setattr(gateway, "_send_request", send_request) + + assert await gateway.mark_auto_repaired("APR-1", "bounded", 10) is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method_name", ["incident_update", "grouped_digest"]) +async def test_lifecycle_no_send_does_not_write_dedup_receipt( + monkeypatch: pytest.MonkeyPatch, + method_name: str, +) -> None: + redis = FakeRedis({"tg_msg:INC-1": "42"}) + gateway = TelegramGateway() + + async def send_request(method: str, _payload: dict): + if method == "sendMessage": + return dict(NO_SEND_RECEIPT) + return {"ok": True} + + monkeypatch.setattr(gateway_module, "get_redis", lambda: redis) + monkeypatch.setattr(gateway, "_send_request", send_request) + + if method_name == "incident_update": + result = await gateway.append_incident_update("INC-1", "healthy") + else: + result = await gateway.append_grouped_alert_digest( + incident_id="INC-1", + group_key="group-1", + digest_text="digest", + ) + + assert result is False + assert redis.set_calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method_name", ["incident_update", "grouped_digest"]) +async def test_lifecycle_writes_dedup_only_after_affirmative_ack( + monkeypatch: pytest.MonkeyPatch, + method_name: str, +) -> None: + redis = FakeRedis({"tg_msg:INC-1": "42"}) + gateway = TelegramGateway() + + async def send_request(method: str, _payload: dict): + if method == "sendMessage": + return dict(SENT_RECEIPT) + return {"ok": True} + + monkeypatch.setattr(gateway_module, "get_redis", lambda: redis) + monkeypatch.setattr(gateway, "_send_request", send_request) + + if method_name == "incident_update": + result = await gateway.append_incident_update("INC-1", "healthy") + else: + result = await gateway.append_grouped_alert_digest( + incident_id="INC-1", + group_key="group-1", + digest_text="digest", + ) + + assert result is True + assert len(redis.set_calls) == 1 + assert redis.set_calls[0][1] == "provider_ack" + + +@pytest.mark.asyncio +async def test_analyzing_placeholder_does_not_return_or_log_sent_on_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + logger = FakeLogger() + gateway = TelegramGateway() + monkeypatch.setattr(gateway_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(gateway_module, "logger", logger) + monkeypatch.setattr( + gateway, + "_send_request", + AsyncMock(return_value=dict(NO_SEND_RECEIPT)), + ) + + result = await gateway.send_analyzing_placeholder("cpu", "api", "medium") + + assert result is None + assert ("info", "analyzing_placeholder_no_send") in logger.events + assert ("info", "analyzing_placeholder_sent") not in logger.events + + +@pytest.mark.asyncio +async def test_approval_card_records_failed_and_never_sent_for_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + logger = FakeLogger() + gateway = TelegramGateway() + executed_params: list[dict] = [] + + class FakeDB: + async def execute(self, _statement, params: dict) -> None: + executed_params.append(params) + + @asynccontextmanager + async def fake_db_context(*_args, **_kwargs): + yield FakeDB() + + monkeypatch.setattr(gateway_module, "logger", logger) + monkeypatch.setattr( + gateway_module, + "_fetch_remediation_summary_for_card", + AsyncMock(return_value=None), + ) + monkeypatch.setattr(gateway, "_build_inline_keyboard", AsyncMock(return_value={})) + monkeypatch.setattr( + gateway, + "_send_request", + AsyncMock(return_value=dict(NO_SEND_RECEIPT)), + ) + monkeypatch.setattr("src.db.base.get_db_context", fake_db_context) + + result = await gateway.send_approval_card( + approval_id="APR-1", + risk_level="medium", + resource_name="api", + root_cause="cpu", + suggested_action="inspect", + notification_type="TYPE-3", + ) + + assert result == NO_SEND_RECEIPT + assert executed_params and executed_params[0]["ds"] == "failed" + assert ("warning", "telegram_approval_card_no_send") in logger.events + assert ("info", "telegram_approval_card_sent") not in logger.events + + +@pytest.mark.asyncio +async def test_html_callback_no_send_exhausts_fallback_and_records_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value=dict(NO_SEND_RECEIPT)) + record_failure = AsyncMock() + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + + await gateway._send_html_line_message( + ["reply"], + failure_context="test", + inbound_chat_id="room", + inbound_message_id=42, + incident_id="INC-1", + ) + + assert send_request.await_count == 3 + record_failure.assert_awaited_once() + statuses = [ + call.args[1].get(gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY, {}) + .get("callback_reply", {}) + .get("status") + for call in send_request.await_args_list + ] + assert statuses == [ + "callback_reply_attempt", + "callback_reply_fallback_attempt", + "callback_reply_rescue_attempt", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reply_kind", ["advisory", "category"]) +async def test_interactive_reply_no_send_logs_failure_not_sent( + monkeypatch: pytest.MonkeyPatch, + reply_kind: str, +) -> None: + from src.services import ai_advisory_helpers, callback_dispatcher + + logger = FakeLogger() + gateway = TelegramGateway() + monkeypatch.setattr(gateway_module, "logger", logger) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + monkeypatch.setattr( + gateway, + "_send_request", + AsyncMock(return_value=dict(NO_SEND_RECEIPT)), + ) + + if reply_kind == "advisory": + monkeypatch.setattr( + ai_advisory_helpers, + "handle_ai_advisory_callback", + AsyncMock( + return_value={ + "feedback_text": "ack", + "success": True, + } + ), + ) + await gateway._handle_ai_advisory_action( + "ai_advisory_view", + "cost:ADV-1", + "callback-1", + 1, + "operator", + {"id": 1}, + inbound_chat_id="room", + inbound_message_id=42, + ) + failed_event = "ai_advisory_group_reply_failed" + sent_event = "ai_advisory_group_reply_sent" + else: + class Repo: + async def get_by_id(self, _incident_id: str): + return None + + monkeypatch.setattr( + callback_dispatcher, + "get_action_spec", + lambda _action: SimpleNamespace(emoji="info"), + ) + monkeypatch.setattr( + callback_dispatcher, + "dispatch_action", + AsyncMock( + return_value=SimpleNamespace( + result_text="result", + success=True, + duration_ms=1.0, + ) + ), + ) + monkeypatch.setattr( + "src.repositories.incident_repository.get_incident_repository", + lambda: Repo(), + ) + await gateway._dispatch_category_action( + "callback-1", + "history", + "INC-1", + 1, + inbound_chat_id="room", + inbound_message_id=42, + ) + failed_event = "category_action_reply_failed" + sent_event = "category_action_reply_sent" + + assert ("warning", failed_event) in logger.events + assert ("info", sent_event) not in logger.events + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("notification_type", "alert_category"), + [("generic", ""), ("TYPE-4D", ""), ("TYPE-8M", "alertchain_health")], +) +async def test_background_no_send_never_confirms_or_writes_sent( + monkeypatch: pytest.MonkeyPatch, + notification_type: str, + alert_category: str, +) -> None: + calls: list[dict] = [] + + class Gateway: + async def send_approval_card(self, **kwargs): + calls.append(kwargs) + return dict(NO_SEND_RECEIPT) + + async def send_drift_card(self, **kwargs): + calls.append(kwargs) + return dict(NO_SEND_RECEIPT) + + async def send_meta_alert(self, **kwargs): + calls.append(kwargs) + return dict(NO_SEND_RECEIPT) + + async def delete_message(self, *_args, **_kwargs): + raise AssertionError("placeholder must not be deleted before delivery") + + class ApprovalService: + async def mark_telegram_confirmed(self, *_args, **_kwargs): + raise AssertionError("no-send must not be marked confirmed") + + monkeypatch.setattr(webhooks.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(webhooks, "get_telegram_gateway", lambda: Gateway()) + monkeypatch.setattr(webhooks, "get_approval_service", lambda: ApprovalService()) + + await webhooks._push_to_telegram_background( + approval_id="APR-1", + risk_level="medium", + resource_name="api", + root_cause="cpu", + suggested_action="inspect", + estimated_downtime="0s", + notification_type=notification_type, + alert_category=alert_category, + fingerprint="fp", + placeholder_message_id=42, + ) + + assert len(calls) == 1 + if notification_type == "generic": + assert calls[0]["notification_type"] == "TYPE-3" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("module_name", ["sentry", "signoz"]) +async def test_monitoring_webhooks_use_type3_and_do_not_false_log_sent( + monkeypatch: pytest.MonkeyPatch, + module_name: str, +) -> None: + logger = FakeLogger() + captured: list[dict] = [] + + class Gateway: + async def initialize(self) -> bool: + return True + + async def send_approval_card(self, **kwargs): + captured.append(kwargs) + return dict(NO_SEND_RECEIPT) + + if module_name == "sentry": + monkeypatch.setattr(sentry_webhook, "logger", logger) + monkeypatch.setattr(sentry_webhook, "get_telegram_gateway", lambda: Gateway()) + await sentry_webhook.send_sentry_telegram_alert( + {"title": "error", "culprit": "api.py", "level": "error"}, + None, + "APR-1", + ) + sent_event = "sentry_telegram_sent" + no_send_event = "sentry_telegram_no_send" + else: + monkeypatch.setattr(signoz_webhook, "logger", logger) + monkeypatch.setattr(signoz_webhook, "get_telegram_gateway", lambda: Gateway()) + await signoz_webhook.send_signoz_telegram( + "APR-1", + "INC-1", + "HighCPU", + {"service": "api"}, + {"summary": "cpu"}, + "warning", + ) + sent_event = "signoz_telegram_sent" + no_send_event = "signoz_telegram_no_send" + + assert captured[0]["notification_type"] == "TYPE-3" + assert ("warning", no_send_event) in logger.events + assert ("info", sent_event) not in logger.events + + +@pytest.mark.asyncio +async def test_dev_test_push_returns_false_for_structured_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[dict] = [] + + class Gateway: + async def send_approval_card(self, **kwargs): + captured.append(kwargs) + return dict(NO_SEND_RECEIPT) + + monkeypatch.setattr(telegram.settings, "ENVIRONMENT", "dev") + monkeypatch.setattr(telegram, "get_telegram_gateway", lambda: Gateway()) + + response = await telegram.test_push( + telegram.TestPushRequest(approval_id="APR-1") + ) + + assert response["ok"] is False + assert response["message"] == "Test push no-send" + assert captured[0]["notification_type"] == "TYPE-3" diff --git a/apps/api/tests/test_telegram_message_templates.py b/apps/api/tests/test_telegram_message_templates.py index ee1b59ce9..e57038bb2 100644 --- a/apps/api/tests/test_telegram_message_templates.py +++ b/apps/api/tests/test_telegram_message_templates.py @@ -503,6 +503,11 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch) outbound_receipts.append(kwargs) monkeypatch.setattr(TelegramGateway, "api_url", property(lambda _self: "https://telegram.test/botx")) + monkeypatch.setattr( + telegram_gateway_module.settings, + "SRE_GROUP_CHAT_ID", + "chat", + ) monkeypatch.setattr( telegram_gateway_module, "_agent99_ai_alert_card_mirror_configured", @@ -526,11 +531,18 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch) "direct ref verifier readback" ), "parse_mode": "MarkdownV2", + "_awoooi_canonical_telegram_route": { + "product_id": "awoooi", + "signal_family": "shared_infrastructure", + "severity": "P1", + }, }, ) await asyncio.sleep(0) - assert result == {"ok": True, "result": {"message_id": 991}} + assert result["ok"] is True + assert result["result"] == {"message_id": 991} + assert result["_awoooi_canonical_route_receipt"]["decision"] == "allowed" assert mirrored assert mirrored[0]["source"] == "send_request" assert "ai_automation_alert_card_v1" in mirrored[0]["text"] @@ -1806,7 +1818,13 @@ async def test_send_request_strips_awooop_callback_metadata_before_telegram_api( { "chat_id": "chat", "text": "事件歷史統計 INC-20260513-79ED5E", + "reply_to_message_id": 123, "_skip_incident_thread_reply": True, + "_awoooi_non_monitoring_interaction": { + "interaction_kind": "callback_reply", + "inbound_chat_id": "chat", + "inbound_message_id": 123, + }, "_awooop_source_envelope_extra": { "callback_reply": { "status": "callback_reply_sent", @@ -1866,8 +1884,9 @@ async def test_send_html_line_message_falls_back_to_plain_text_on_parse_error(mo await gateway._send_html_line_message( ["📊 事件歷史統計", "階段: blocked"], - chat_id="chat", failure_context="test_history", + inbound_chat_id="chat", + inbound_message_id=77, reply_markup=reply_markup, ) @@ -1882,7 +1901,7 @@ async def test_send_html_line_message_falls_back_to_plain_text_on_parse_error(mo @pytest.mark.asyncio async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch): - """詳情/歷史 callback reply 要在 AwoooP envelope 標明 sent/fallback 狀態。""" + """Callback envelope stays attempt until affirmative provider delivery.""" sent_requests = [] gateway = TelegramGateway() reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21") @@ -1897,8 +1916,9 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) await gateway._send_html_line_message( ["📊 事件歷史統計", "🔖 INC-20260514-F85F21"], - chat_id="chat", failure_context="incident_history", + inbound_chat_id="chat", + inbound_message_id=77, reply_markup=reply_markup, incident_id="INC-20260514-F85F21", callback_action="history", @@ -1958,7 +1978,7 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) first_extra = first_source_extra["callback_reply"] fallback_extra = fallback_source_extra["callback_reply"] - assert first_extra["status"] == "callback_reply_sent" + assert first_extra["status"] == "callback_reply_attempt" assert first_extra["action"] == "history" assert first_extra["parse_mode"] == "HTML" assert first_source_extra["awooop_status_chain"]["repair_state"] == ( @@ -1969,7 +1989,7 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) assert first_source_extra["km_stale_completion_summary"]["triage"]["flow_stage"] == ( "callback_observed_owner_review_link_missing" ) - assert fallback_extra["status"] == "callback_reply_fallback_sent" + assert fallback_extra["status"] == "callback_reply_fallback_attempt" assert fallback_extra["incident_id"] == "INC-20260514-F85F21" assert fallback_extra["parse_mode"] == "plain_text" assert fallback_source_extra["awooop_status_chain"]["next_step"] == ( @@ -1979,6 +1999,37 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) "ai_lead_agent" ] == "Hermes" + acknowledged = ( + telegram_gateway_module._acknowledge_callback_reply_source_envelope( + fallback_source_extra, + delivery_result={"ok": True}, + provider_message_id="88", + ) + ) + assert acknowledged["callback_reply"]["status"] == ( + "callback_reply_fallback_sent" + ) + assert acknowledged["callback_reply"]["provider_delivery_acknowledged"] is True + assert acknowledged["callback_reply"]["provider_message_id"] == "88" + + structured_no_send = ( + telegram_gateway_module._acknowledge_callback_reply_source_envelope( + fallback_source_extra, + delivery_result={ + "ok": True, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + }, + provider_message_id="88", + ) + ) + assert structured_no_send["callback_reply"]["status"] == ( + "callback_reply_fallback_attempt" + ) + assert "provider_delivery_acknowledged" not in structured_no_send[ + "callback_reply" + ] + @pytest.mark.asyncio async def test_send_html_line_message_uses_rescue_when_markup_fallback_fails(monkeypatch): @@ -1997,8 +2048,9 @@ async def test_send_html_line_message_uses_rescue_when_markup_fallback_fails(mon await gateway._send_html_line_message( ["📊 事件歷史統計", "🔖 INC-20260514-F85F21"], - chat_id="chat", failure_context="test_history", + inbound_chat_id="chat", + inbound_message_id=77, reply_markup=reply_markup, ) @@ -2027,8 +2079,9 @@ async def test_send_html_line_message_attaches_awooop_markup_to_first_chunk(monk f"INC-20260514-F85F21 trace line {index:03d}" for index in range(80) ], - chat_id="chat", failure_context="test_history", + inbound_chat_id="chat", + inbound_message_id=77, reply_markup=reply_markup, ) @@ -2048,8 +2101,12 @@ async def test_info_callback_sends_history_when_answer_callback_is_stale(monkeyp async def stale_answer(*_args, **_kwargs): raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400") - async def fake_history(incident_id: str): + async def fake_history(incident_id: str, **context): sent_history.append(incident_id) + assert context == { + "inbound_chat_id": "chat", + "inbound_message_id": 789, + } monkeypatch.setattr(gateway, "_answer_callback", stale_answer) monkeypatch.setattr(gateway, "_send_incident_history", fake_history) @@ -2059,6 +2116,7 @@ async def test_info_callback_sends_history_when_answer_callback_is_stale(monkeyp callback_data="history:INC-20260513-79ED5E", user_id=123456, message_id=789, + chat_id="chat", ) assert result["success"] is True @@ -2083,6 +2141,9 @@ async def test_send_notification_retries_plain_text_on_html_parse_400(monkeypatc result = await gateway.send_notification( "⚠️ 無法取得歷史統計: broken", chat_id="chat", + interaction_kind="callback_reply", + inbound_chat_id="chat", + inbound_message_id=77, ) assert result == {"ok": True} @@ -2104,7 +2165,13 @@ async def test_send_notification_long_html_uses_plain_text_without_cutting_tags( monkeypatch.setattr(gateway, "_send_request", fake_send_request) long_text = "📊 事件歷史統計\n告警鍵: " + ("node<188>&" * 90) + "" - await gateway.send_notification(long_text, chat_id="chat") + await gateway.send_notification( + long_text, + chat_id="chat", + interaction_kind="callback_reply", + inbound_chat_id="chat", + inbound_message_id=77, + ) assert len(sent_requests) == 1 payload = sent_requests[0][1] diff --git a/docs/adr/ADR-093-telegram-group-migration.md b/docs/adr/ADR-093-telegram-group-migration.md index c2a80d42a..1e32585f8 100644 --- a/docs/adr/ADR-093-telegram-group-migration.md +++ b/docs/adr/ADR-093-telegram-group-migration.md @@ -1,5 +1,9 @@ # ADR-093: Telegram 告警全面遷移至 SRE 戰情室群組 +> **2026-07-14 sunset**：本 ADR 的「所有產品/未知類型預設進 SRE 群組」已由 `global_product_governance_v2` 取代。 +> `superseded_by=global_product_governance_v2`；owner=`sre_shared_runtime`；expiry=`2026-07-14T14:35:26+08:00`；replacement=`docs/security/telegram-canonical-routing-registry.snapshot.json`；verifier=`apps/api/tests/test_telegram_canonical_routing_registry.py`。 +> 保留的唯一行為是 allowlisted AWOOOI shared infra/security/DR/P0-P1 lifecycle；未知 route、raw P2/P3、marketing 與跨產品 fallback 一律 blocked。 + > **狀態**: ✅ 已全面切換 > **日期**: 2026-04-24 > **決策者**: 統帥 + 12-Agent 全景分析團隊（onboarder / debugger / db-expert / tool-expert / web-researcher / planner / critic / frontend-designer / fullstack-engineer / refactor-specialist / migration-engineer / vuln-verifier） @@ -8,7 +12,7 @@ 原 Telegram 告警雙軌路由設計： - 所有告警類（TYPE-3/4/4D/8M）預設發私人 DM `OPENCLAW_TG_CHAT_ID` -- 僅心跳報告、週月報、AI 三頭分析發群組 `SRE_GROUP_CHAT_ID=-1003711974679` +- 僅心跳報告、週月報、AI 三頭分析發群組 alias `SRE_GROUP_CHAT_ID`（numeric ID 不進文件） - `_interactive_types` 黑名單曾把互動式告警擋在群組外 使用者需求：把**所有**告警（含需人工簽核的 TYPE-3/4/4D/8M）遷到 AwoooI SRE 戰情室群組（4 名成員），以便團隊共同監察。 diff --git a/docs/schemas/telegram_canonical_routing_registry_v1.schema.json b/docs/schemas/telegram_canonical_routing_registry_v1.schema.json new file mode 100644 index 000000000..e1946ad80 --- /dev/null +++ b/docs/schemas/telegram_canonical_routing_registry_v1.schema.json @@ -0,0 +1,309 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://awoooi.wooo.work/schemas/telegram_canonical_routing_registry_v1.schema.json", + "title": "Telegram Canonical Routing Registry v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "generated_at", + "status", + "mode", + "supersedes", + "policy", + "destination_roles", + "products", + "routes", + "rollups", + "execution_boundaries" + ], + "properties": { + "schema_version": { + "const": "telegram_canonical_routing_registry_v1" + }, + "generated_at": {"type": "string", "format": "date-time"}, + "status": {"const": "source_policy_ready_runtime_not_applied"}, + "mode": {"const": "source_only_no_secret_no_telegram_send_no_route_change"}, + "supersedes": { + "type": "object", + "additionalProperties": false, + "required": [ + "policy_ref", + "superseded_behavior", + "superseded_by", + "owner", + "expiry", + "replacement", + "verifier" + ], + "properties": { + "policy_ref": {"type": "string", "minLength": 1}, + "superseded_behavior": {"type": "string", "minLength": 1}, + "superseded_by": {"const": "global_product_governance_v2"}, + "owner": {"type": "string", "minLength": 1}, + "expiry": {"type": "string", "format": "date-time"}, + "replacement": {"type": "string", "minLength": 1}, + "verifier": {"type": "string", "minLength": 1} + } + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "default_decision", + "unknown_product_decision", + "unknown_signal_family_decision", + "unknown_severity_decision", + "unknown_destination_decision", + "legacy_runtime_resolver", + "legacy_shared_sre_allowed_types", + "legacy_shared_sre_blocked_types", + "raw_chat_id_allowed", + "shared_sre_rule", + "shared_sre_forbidden" + ], + "properties": { + "default_decision": {"const": "block"}, + "unknown_product_decision": {"const": "block"}, + "unknown_signal_family_decision": {"const": "block"}, + "unknown_severity_decision": {"const": "block"}, + "unknown_destination_decision": {"const": "block"}, + "legacy_runtime_resolver": {"type": "string", "minLength": 1}, + "legacy_shared_sre_allowed_types": { + "type": "array", + "minItems": 7, + "maxItems": 7, + "uniqueItems": true, + "items": { + "enum": ["TYPE-2", "TYPE-3", "TYPE-4", "TYPE-4D", "TYPE-5S", "TYPE-7E", "TYPE-8M"] + } + }, + "legacy_shared_sre_blocked_types": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "uniqueItems": true, + "items": {"enum": ["TYPE-1", "TYPE-6B", "UNKNOWN"]} + }, + "raw_chat_id_allowed": {"const": false}, + "shared_sre_rule": {"type": "string", "minLength": 1}, + "shared_sre_forbidden": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + } + } + }, + "destination_roles": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "propertyNames": { + "not": { + "pattern": "(?:[Cc][Hh][Aa][Tt][_-]?[Ii][Dd]|[Tt][Oo][Kk][Ee][Nn]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Aa][Uu][Tt][Hh][Oo][Rr][Ii][Zz][Aa][Tt][Ii][Oo][Nn]|[Ww][Ee][Bb][Hh][Oo][Oo][Kk])" + } + }, + "required": [ + "visible_label", + "proven_runtime_alias", + "runtime_alias_evidence", + "chat_alias", + "bot_alias", + "bot_owner", + "monitoring_owner", + "role", + "monitoring_egress_policy", + "runtime_identifier_in_registry" + ], + "properties": { + "visible_label": { + "type": "string", + "minLength": 1, + "pattern": "^(?!-?[0-9]{8,}$).+$" + }, + "proven_runtime_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "runtime_alias_evidence": { + "enum": [ + "proven_runtime_receipt", + "source_config_only", + "configured_unreachable", + "visible_label_only" + ] + }, + "chat_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "bot_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "bot_owner": {"type": "string", "minLength": 1}, + "monitoring_owner": {"type": "boolean"}, + "role": {"type": "string", "minLength": 1}, + "monitoring_egress_policy": {"type": "string", "minLength": 1}, + "runtime_identifier_in_registry": {"const": false} + } + } + }, + "products": { + "type": "array", + "minItems": 11, + "maxItems": 11, + "items": { + "type": "object", + "additionalProperties": false, + "propertyNames": { + "not": { + "pattern": "(?:[Cc][Hh][Aa][Tt][_-]?[Ii][Dd]|[Tt][Oo][Kk][Ee][Nn]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Aa][Uu][Tt][Hh][Oo][Rr][Ii][Zz][Aa][Tt][Ii][Oo][Nn]|[Ww][Ee][Bb][Hh][Oo][Oo][Kk])" + } + }, + "required": [ + "product_id", + "display_name", + "owner", + "source_audit_status", + "egress_status" + ], + "properties": { + "product_id": {"type": "string", "minLength": 1}, + "display_name": {"type": "string", "minLength": 1}, + "owner": {"type": "string", "minLength": 1}, + "source_audit_status": {"type": "string", "minLength": 1}, + "egress_status": { + "enum": ["allowed", "blocked", "disabled", "not_implemented"] + } + } + } + }, + "routes": { + "type": "array", + "minItems": 11, + "items": { + "type": "object", + "additionalProperties": false, + "propertyNames": { + "not": { + "pattern": "(?:[Cc][Hh][Aa][Tt][_-]?[Ii][Dd]|[Tt][Oo][Kk][Ee][Nn]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Aa][Uu][Tt][Hh][Oo][Rr][Ii][Zz][Aa][Tt][Ii][Oo][Nn]|[Ww][Ee][Bb][Hh][Oo][Oo][Kk])" + } + }, + "required": [ + "route_id", + "product_id", + "signal_family", + "severity", + "scope", + "bot_alias", + "chat_alias", + "allowed_destination", + "receipt_backend", + "owner", + "egress_status" + ], + "properties": { + "route_id": {"type": "string", "minLength": 1}, + "product_id": {"type": "string", "minLength": 1}, + "signal_family": {"type": "string", "minLength": 1}, + "severity": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"enum": ["P0", "P1", "P2", "P3"]} + }, + "scope": {"enum": ["shared", "product"]}, + "bot_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "chat_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "allowed_destination": { + "type": "string", + "pattern": "^(?:blocked|telegram_chat_alias:[a-z0-9_]+)$" + }, + "receipt_backend": { + "type": "string", + "minLength": 1, + "pattern": "^(?!-?[0-9]{8,}$).+$" + }, + "owner": {"type": "string", "minLength": 1}, + "egress_status": { + "enum": ["allowed", "blocked", "disabled", "not_implemented"] + }, + "blocked_reason": {"type": "string", "minLength": 1} + }, + "allOf": [ + { + "if": {"properties": {"egress_status": {"const": "allowed"}}}, + "then": { + "properties": { + "allowed_destination": { + "pattern": "^telegram_chat_alias:[a-z0-9_]+$" + } + } + }, + "else": { + "required": ["blocked_reason"], + "properties": {"allowed_destination": {"const": "blocked"}} + } + } + ] + } + }, + "rollups": { + "type": "object", + "additionalProperties": false, + "required": [ + "product_count", + "route_count", + "allowed_route_count", + "blocked_route_count", + "disabled_route_count", + "not_implemented_route_count", + "shared_sre_allowed_route_count", + "raw_numeric_destination_count", + "runtime_route_change_count", + "telegram_send_count" + ], + "properties": { + "product_count": {"const": 11}, + "route_count": {"type": "integer", "minimum": 11}, + "allowed_route_count": {"type": "integer", "minimum": 0}, + "blocked_route_count": {"type": "integer", "minimum": 0}, + "disabled_route_count": {"type": "integer", "minimum": 0}, + "not_implemented_route_count": {"type": "integer", "minimum": 0}, + "shared_sre_allowed_route_count": {"type": "integer", "minimum": 0}, + "raw_numeric_destination_count": {"const": 0}, + "runtime_route_change_count": {"const": 0}, + "telegram_send_count": {"const": 0} + } + }, + "execution_boundaries": { + "type": "object", + "additionalProperties": false, + "required": [ + "telegram_send_allowed", + "bot_api_call_allowed", + "secret_value_read_allowed", + "raw_chat_id_storage_allowed", + "runtime_route_change_allowed", + "production_deploy_allowed" + ], + "properties": { + "telegram_send_allowed": {"const": false}, + "bot_api_call_allowed": {"const": false}, + "secret_value_read_allowed": {"const": false}, + "raw_chat_id_storage_allowed": {"const": false}, + "runtime_route_change_allowed": {"const": false}, + "production_deploy_allowed": {"const": false} + } + } + } +} diff --git a/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md new file mode 100644 index 000000000..2e30d028a --- /dev/null +++ b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md @@ -0,0 +1,60 @@ +# Telegram 監控告警路由總帳 + +> 狀態：`source_policy_ready_runtime_not_applied` +> 真相邊界：本表把「已證明現況」、「建議目的地」、「目前阻擋」、「source risk」與「receipt 強度」分開。建議不等於 live route；alias 不是 numeric chat ID；本變更沒有送 Telegram、讀 token/chat ID、改 runtime route 或部署。 + +## 判讀規則 + +- `visible label` 只描述使用者畫面中的名稱；`proven runtime alias` 必須另有 source/runtime receipt，兩者不得互相替代。 +- `proven route` 只描述目前可由 source 或 durable receipt 證明的行為，不把畫面名稱或建議當成 live delivery。 +- `recommended route` 是 canonical 產品責任分流；若 bot alias、chat alias 或 durable receipt 未證明，egress 仍是 `blocked`。 +- `receipt strength`：`strong` 代表 durable DB receipt；`weak` 代表 console/source 行為；`none` 代表沒有可驗證的 delivery receipt。 +- 未知 product/signal/severity/destination 一律 no-egress；不再沿用 ADR-093 的 unknown-to-shared-group fallback。 +- `AwoooI SRE 戰情室` 只收 shared infrastructure、security、DR 與 P0/P1 incident lifecycle。raw P2/P3、marketing、成功 heartbeat、未驗證 recommendation 和 bot 對話均不得進入。 + +## 11 產品路由清冊 + +| 產品 | 目前 proven route | 建議 canonical route | blocked unknown / egress status | source risk | receipt strength | +|---|---|---|---|---|---| +| `2026fifa` | source-only monitoring surface；沒有 runtime Telegram binding | 建立專屬 product alias 後，只送 data freshness、recommendation build、public API failure | `blocked`：chat/bot alias 與 durable receipt 均未證明 | generic env 可能誤入其他產品 lane | `none` | +| `agent-bounty-protocol` | `VibeAIAgent` bot-to-group route 已證明，涵蓋 A2A/traffic/treasury/onboarding；durable receipt 仍缺 | `vibe_ai_agent_control_plane`，先收斂到 gateway 與 durable receipt 再開 egress | `blocked`：route 存在但 console/source log 不能代替 durable delivery receipt | 訊息標題仍可能把 Agent Bounty 與 standalone VibeWork 混為同一產品 | `weak`：console/source only | +| `awooogo` | 沒有 runtime Telegram binding，egress disabled | 維持 disabled；未來啟用需先建立獨立 product alias/receipt | `disabled` | 不得用 shared SRE 當缺設定 fallback | `none` | +| `awoooi` | `AwoooI SRE 戰情室` 與 TsenYang monitoring-owner receipt 已證明；`小O_小龍蝦`/`叫小賀` 是 visible label/source-config candidate，不當成 runtime alias proof | TsenYang bot 才能送 canonical shared infra/security/DR/P0-P1 lifecycle；OpenClaw/NemoTron 只可在原訊息 interaction context 下回覆 | shared allowlist `allowed`；raw P2/P3、business/FinOps、marketing `blocked` | 若未驗 sender bot ownership，AI bot 會繞過 canonical alert owner | `strong`（僅 shared monitoring owner） | +| `bitan-pharmacy` | source-only product/container signals；沒有 canonical runtime Telegram binding | 建立 Bitan-owned alias 後才允許 product health/failure | `blocked`：目的地、bot alias、receipt backend 未證明 | cross-product container signal 容易被誤當 AWOOOI shared infra | `none` | +| `clawbot-openclaw` | 目前使用 `tsenyangbot` private lane，形成跨產品 bot ownership drift | 建立 ClawBot-owned bot/chat binding；interactive reply 與 monitoring 分離 | `blocked`：現有 private route 不具 ClawBot canonical ownership | 共用 AWOOOI bot 會造成 polling ownership、回覆與告警歸戶混亂 | `none` | +| `momo-pro` | MOMO bot live；configured chat readback 回 `400 chat_not_found` | 修復 MOMO-owned destination 後，才承接 crawler/freshness、scheduler/sales pipeline | `blocked`：runtime destination unreachable，且無 durable receipt | `WOOO` 只是 private bot lane/inferred，不是已證明 MOMO product group | `none` | +| `stockplatform-v2` | 與 TSENYANG Website 共用 `wooowooowooobot` private lane | 建立 StockPlatform-owned destination，只送 market data freshness、pipeline/API failure | `blocked`：跨產品 shared-private ownership drift，無 durable receipt | 同一 private bot lane 無法清楚歸戶、抑噪與回寫 | `none` | +| `tsenyang-website` | 與 StockPlatform 共用 `wooowooowooobot` private lane | 建立 TSENYANG Website-owned destination；operator interaction 不承接 bulk raw alert | `blocked`：跨產品 shared-private ownership drift，無 durable receipt | site 與 market-data 告警混流，owner/verifier 不可辨 | `none` | +| `vibework` | standalone product 沒有 runtime Telegram binding；`VibeAIAgent` 屬 Agent Bounty | 保持 `not_implemented`；若實作需獨立 product_id、alias 與 receipt | `not_implemented` | 名稱相似會造成錯誤歸戶與重複告警 | `none` | +| `vtuber` | 沒有 runtime Telegram binding，egress disabled | 維持 disabled；未來啟用須先建立 VTuber-owned alias | `disabled` | shared monitor 辨識到 signal 不等於產品已有 sender | `none` | + +## 群組與 BOT 的責任邊界 + +| visible label | proven runtime alias | canonical candidate | 可以承接 | 不可以承接 | +|---|---|---|---|---| +| AwoooI SRE 戰情室 | `awoooi_sre_war_room` | TsenYang monitoring owner | shared infra/security/DR、P0/P1 lifecycle | raw P2/P3、business/FinOps、marketing、成功 heartbeat、各產品 bulk warning | +| 小O_小龍蝦 | `none`（visible label only） | `openclaw_bot` source-config candidate | 只在原 inbound message/thread context 下做診斷回覆 | canonical production alert ownership、root alert fanout | +| 叫小賀 | `none`（source-config only） | `nemotron_bot` candidate | 只在原 inbound message/thread context 下做 challenger/shadow reply | canonical production alert ownership | +| TsenYang private | `none`（visible label only） | `wooowooowooobot_private` candidate | 無；僅保留 ownership drift evidence | 在 ownership reconciliation 前不得視為任何產品 canonical route | +| tsenyangbot private | `none`（source-config only） | `tsenyangbot_private` candidate | 無；目前 ClawBot traffic 只作 drift evidence | 跨產品 monitoring ownership | +| WOOO | `none`（visible label only） | `wooo_private_bot_lane` candidate | 尚未證明任何 product-group ownership | 不得當作 MOMO 或 eWoooc canonical product group | +| VibeAIAgent | `none`（source-config only） | `vibe_ai_agent_control_plane` candidate | Agent Bounty A2A/traffic/treasury/onboarding（補 durable receipt 後） | standalone VibeWork 或其他產品告警 | +| MOMO configured chat | `none`（configured unreachable） | `momo_configured_chat_unresolved` candidate | 無；目前 `chat_not_found` | 在 runtime readback、receipt 與 verifier 通過前不得送 | + +## Source risk 與落地缺口 + +1. ADR-093 的 unknown/default-to-group 與全產品治理 v2 衝突；legacy `resolve_chat_ids` 現在必須先通過 canonical registry。unknown、`TYPE-1` info/success 與 `TYPE-6B` business 全部回空 route，不再進 shared SRE。 +2. Agent Bounty 的 bot-to-`VibeAIAgent` route 已證明，但 direct Bot API 與 console log 仍不是 durable receipt；因此 route 存在、egress 仍 blocked 兩件事必須同時呈現。 +3. MOMO bot live 不等於 destination healthy；目前 `chat_not_found` 是明確 blocked runtime truth。`WOOO` 也只能標為 private/inferred lane，不能升格成 product group。 +4. StockPlatform/TSENYANG Website 共用 private bot，ClawBot 又使用 AWOOOI-owned bot，都是 ownership drift；需各自 owner、gateway、receipt、verifier 後才能開 route。 +5. Bitan、2026FIFA 是 source-only；AwoooGo、VibeWork、VTuber 是 disabled/not-implemented/no binding。它們不得因缺專屬群而 fallback 到 shared SRE。 +6. Gateway 現在會在 provider call 前驗 product/signal/severity、destination alias 與 sender bot alias；非監控的 inbound command/callback/thread reply 使用獨立 interaction classifier，不與監控 allowlist 混用。 + +## Machine-readable truth + +- Registry：`docs/security/telegram-canonical-routing-registry.snapshot.json` +- Schema：`docs/schemas/telegram_canonical_routing_registry_v1.schema.json` +- Resolver：`apps/api/src/services/notification_matrix.py` +- Tests：`apps/api/tests/test_telegram_canonical_routing_registry.py` + +Production apply 前仍需同一 route change run 留下 source diff、check-mode、bounded apply、durable delivery receipt、post-verifier、rollback/no-write terminal 與 learning/writeback；本 source-only 交付不代表 runtime route 已更動。 diff --git a/docs/security/telegram-canonical-routing-registry.snapshot.json b/docs/security/telegram-canonical-routing-registry.snapshot.json new file mode 100644 index 000000000..d89ea55d0 --- /dev/null +++ b/docs/security/telegram-canonical-routing-registry.snapshot.json @@ -0,0 +1,522 @@ +{ + "schema_version": "telegram_canonical_routing_registry_v1", + "generated_at": "2026-07-14T15:50:04+08:00", + "status": "source_policy_ready_runtime_not_applied", + "mode": "source_only_no_secret_no_telegram_send_no_route_change", + "supersedes": { + "policy_ref": "docs/adr/ADR-093-telegram-group-migration.md", + "superseded_behavior": "unknown_or_cross_product_alert_defaults_to_shared_sre_group", + "superseded_by": "global_product_governance_v2", + "owner": "sre_shared_runtime", + "expiry": "2026-07-14T14:35:26+08:00", + "replacement": "this canonical registry and notification_matrix fail-closed resolver", + "verifier": "apps/api/tests/test_telegram_canonical_routing_registry.py" + }, + "policy": { + "default_decision": "block", + "unknown_product_decision": "block", + "unknown_signal_family_decision": "block", + "unknown_severity_decision": "block", + "unknown_destination_decision": "block", + "legacy_runtime_resolver": "resolve_chat_ids_must_pass_canonical_registry", + "legacy_shared_sre_allowed_types": [ + "TYPE-2", + "TYPE-3", + "TYPE-4", + "TYPE-4D", + "TYPE-5S", + "TYPE-7E", + "TYPE-8M" + ], + "legacy_shared_sre_blocked_types": ["TYPE-1", "TYPE-6B", "UNKNOWN"], + "raw_chat_id_allowed": false, + "shared_sre_rule": "Only shared infrastructure, security incident, disaster recovery, and P0/P1 incident lifecycle signals may use the AwoooI SRE war room alias.", + "shared_sre_forbidden": [ + "raw product P2/P3 monitoring", + "marketing", + "success heartbeat", + "unverified recommendation", + "bot conversation" + ] + }, + "destination_roles": [ + { + "visible_label": "AwoooI SRE 戰情室", + "proven_runtime_alias": "awoooi_sre_war_room", + "runtime_alias_evidence": "proven_runtime_receipt", + "chat_alias": "awoooi_sre_war_room", + "bot_alias": "tsenyang_bot", + "bot_owner": "sre_shared_runtime", + "monitoring_owner": true, + "role": "shared infrastructure, security, DR, and P0/P1 incident lifecycle", + "monitoring_egress_policy": "allowlist_only", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "TsenYang", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "visible_label_only", + "chat_alias": "wooowooowooobot_private", + "bot_alias": "wooowooowooobot", + "bot_owner": "unverified", + "monitoring_owner": false, + "role": "visible private label only; StockPlatform and TSENYANG Website runtime alias ownership is not proven", + "monitoring_egress_policy": "blocked_cross_product_ownership_drift", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "小O_小龍蝦", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "visible_label_only", + "chat_alias": "awoooi_sre_war_room", + "bot_alias": "openclaw_bot", + "bot_owner": "ai_agent_runtime", + "monitoring_owner": false, + "role": "visible group label mapped to a source-configured OpenClaw reply candidate; runtime alias is not proven by this label", + "monitoring_egress_policy": "interaction_reply_only_no_monitoring_ownership", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "叫小賀", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "source_config_only", + "chat_alias": "awoooi_sre_war_room", + "bot_alias": "nemotron_bot", + "bot_owner": "ai_agent_runtime", + "monitoring_owner": false, + "role": "source-configured NemoTron reply candidate; visible label alone is not runtime proof", + "monitoring_egress_policy": "shadow_reply_only_no_production_alert_ownership", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "WOOO", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "visible_label_only", + "chat_alias": "wooo_private_bot_lane", + "bot_alias": "none", + "bot_owner": "unverified", + "monitoring_owner": false, + "role": "observed private bot lane; product-group ownership is inferred and not proven", + "monitoring_egress_policy": "blocked_not_a_proven_product_group", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "VibeAIAgent", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "source_config_only", + "chat_alias": "vibe_ai_agent_control_plane", + "bot_alias": "vibe_ai_agent_bot", + "bot_owner": "agent_bounty_product_ops", + "monitoring_owner": false, + "role": "proven Agent Bounty bot-to-group route for A2A task, traffic, treasury, and onboarding signals", + "monitoring_egress_policy": "blocked_until_durable_receipt_backend_is_verified", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "tsenyangbot private", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "source_config_only", + "chat_alias": "tsenyangbot_private", + "bot_alias": "tsenyang_bot", + "bot_owner": "sre_shared_runtime", + "monitoring_owner": false, + "role": "observed ClawBot private lane using an AWOOOI-owned bot binding", + "monitoring_egress_policy": "blocked_cross_product_bot_ownership_drift", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "MOMO configured chat", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "configured_unreachable", + "chat_alias": "momo_configured_chat_unresolved", + "bot_alias": "momo_bot", + "bot_owner": "momo_product_ops", + "monitoring_owner": false, + "role": "MOMO bot is live but the configured destination currently returns chat_not_found", + "monitoring_egress_policy": "blocked_runtime_destination_unreachable", + "runtime_identifier_in_registry": false + } + ], + "products": [ + { + "product_id": "2026fifa", + "display_name": "2026 FIFA World Cup", + "owner": "2026fifa_product_ops", + "source_audit_status": "source_only_no_runtime_telegram_binding", + "egress_status": "blocked" + }, + { + "product_id": "agent-bounty-protocol", + "display_name": "Agent Bounty Protocol", + "owner": "agent_bounty_product_ops", + "source_audit_status": "vibe_ai_agent_bot_to_group_route_proven_durable_receipt_gap", + "egress_status": "blocked" + }, + { + "product_id": "awooogo", + "display_name": "AwoooGo", + "owner": "awooogo_product_ops", + "source_audit_status": "no_runtime_telegram_binding_egress_disabled", + "egress_status": "disabled" + }, + { + "product_id": "awoooi", + "display_name": "AWOOOI / IWOOOS", + "owner": "sre_shared_runtime", + "source_audit_status": "shared_group_and_tsenyang_monitoring_owner_receipt_proven_openclaw_nemotron_labels_not_runtime_proof", + "egress_status": "allowed" + }, + { + "product_id": "bitan-pharmacy", + "display_name": "Bitan Pharmacy", + "owner": "bitan_product_ops", + "source_audit_status": "source_only_no_canonical_runtime_telegram_binding", + "egress_status": "blocked" + }, + { + "product_id": "clawbot-openclaw", + "display_name": "Clawbot / OpenClaw", + "owner": "ai_agent_runtime", + "source_audit_status": "tsenyangbot_private_route_observed_cross_product_bot_ownership_drift", + "egress_status": "blocked" + }, + { + "product_id": "momo-pro", + "display_name": "MOMO PRO / eWoooc", + "owner": "momo_product_ops", + "source_audit_status": "momo_bot_live_configured_chat_returns_chat_not_found", + "egress_status": "blocked" + }, + { + "product_id": "stockplatform-v2", + "display_name": "StockPlatform v2", + "owner": "stockplatform_product_ops", + "source_audit_status": "wooowooowooobot_private_route_shared_with_tsenyang_site_ownership_drift", + "egress_status": "blocked" + }, + { + "product_id": "tsenyang-website", + "display_name": "TSENYANG Website", + "owner": "tsenyang_site_ops", + "source_audit_status": "wooowooowooobot_private_route_shared_with_stockplatform_ownership_drift", + "egress_status": "blocked" + }, + { + "product_id": "vibework", + "display_name": "VibeWork", + "owner": "vibework_product_ops", + "source_audit_status": "no_runtime_telegram_binding_agent_bounty_is_separate_owner", + "egress_status": "not_implemented" + }, + { + "product_id": "vtuber", + "display_name": "VTuber", + "owner": "vtuber_product_ops", + "source_audit_status": "no_runtime_telegram_binding_egress_disabled", + "egress_status": "disabled" + } + ], + "routes": [ + { + "route_id": "2026fifa-product-monitoring", + "product_id": "2026fifa", + "signal_family": "data_freshness_and_recommendation", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "unassigned", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "2026fifa_product_ops", + "egress_status": "blocked", + "blocked_reason": "canonical_product_destination_and_durable_receipt_backend_missing" + }, + { + "route_id": "agent-bounty-a2a-task-lifecycle", + "product_id": "agent-bounty-protocol", + "signal_family": "a2a_task_lifecycle", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "vibe_ai_agent_bot", + "chat_alias": "vibe_ai_agent_control_plane", + "allowed_destination": "blocked", + "receipt_backend": "console_log_only_unverified", + "owner": "agent_bounty_product_ops", + "egress_status": "blocked", + "blocked_reason": "direct_bot_api_and_durable_delivery_receipt_gap" + }, + { + "route_id": "agent-bounty-traffic-security", + "product_id": "agent-bounty-protocol", + "signal_family": "traffic_security", + "severity": ["P0", "P1", "P2"], + "scope": "product", + "bot_alias": "vibe_ai_agent_bot", + "chat_alias": "vibe_ai_agent_control_plane", + "allowed_destination": "blocked", + "receipt_backend": "console_log_only_unverified", + "owner": "agent_bounty_product_ops", + "egress_status": "blocked", + "blocked_reason": "direct_bot_api_and_durable_delivery_receipt_gap" + }, + { + "route_id": "agent-bounty-treasury-onboarding", + "product_id": "agent-bounty-protocol", + "signal_family": "treasury_and_onboarding", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "vibe_ai_agent_bot", + "chat_alias": "vibe_ai_agent_control_plane", + "allowed_destination": "blocked", + "receipt_backend": "console_log_only_unverified", + "owner": "agent_bounty_product_ops", + "egress_status": "blocked", + "blocked_reason": "direct_bot_api_and_durable_delivery_receipt_gap" + }, + { + "route_id": "awooogo-disabled", + "product_id": "awooogo", + "signal_family": "*", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "awooogo_product_ops", + "egress_status": "disabled", + "blocked_reason": "telegram_egress_disabled" + }, + { + "route_id": "awoooi-shared-infrastructure", + "product_id": "awoooi", + "signal_family": "shared_infrastructure", + "severity": ["P0", "P1"], + "scope": "shared", + "bot_alias": "tsenyang_bot", + "chat_alias": "awoooi_sre_war_room", + "allowed_destination": "telegram_chat_alias:awoooi_sre_war_room", + "receipt_backend": "awoooi.alert_operation_log+telegram_outbound_receipts", + "owner": "sre_shared_runtime", + "egress_status": "allowed" + }, + { + "route_id": "awoooi-security-incident", + "product_id": "awoooi", + "signal_family": "security_incident", + "severity": ["P0", "P1"], + "scope": "shared", + "bot_alias": "tsenyang_bot", + "chat_alias": "awoooi_sre_war_room", + "allowed_destination": "telegram_chat_alias:awoooi_sre_war_room", + "receipt_backend": "awoooi.alert_operation_log+telegram_outbound_receipts", + "owner": "security_runtime", + "egress_status": "allowed" + }, + { + "route_id": "awoooi-disaster-recovery", + "product_id": "awoooi", + "signal_family": "disaster_recovery", + "severity": ["P0", "P1"], + "scope": "shared", + "bot_alias": "tsenyang_bot", + "chat_alias": "awoooi_sre_war_room", + "allowed_destination": "telegram_chat_alias:awoooi_sre_war_room", + "receipt_backend": "awoooi.alert_operation_log+telegram_outbound_receipts", + "owner": "dr_runtime", + "egress_status": "allowed" + }, + { + "route_id": "awoooi-incident-lifecycle", + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": ["P0", "P1"], + "scope": "shared", + "bot_alias": "tsenyang_bot", + "chat_alias": "awoooi_sre_war_room", + "allowed_destination": "telegram_chat_alias:awoooi_sre_war_room", + "receipt_backend": "awoooi.alert_operation_log+telegram_outbound_receipts", + "owner": "sre_shared_runtime", + "egress_status": "allowed" + }, + { + "route_id": "awoooi-raw-product-monitoring", + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": ["P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "awoooi.alert_operation_log", + "owner": "product_route_registry", + "egress_status": "blocked", + "blocked_reason": "raw_p2_p3_must_not_use_shared_sre_destination" + }, + { + "route_id": "awoooi-marketing", + "product_id": "awoooi", + "signal_family": "marketing", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "product_route_registry", + "egress_status": "blocked", + "blocked_reason": "marketing_must_not_use_monitoring_egress" + }, + { + "route_id": "awoooi-business-finops", + "product_id": "awoooi", + "signal_family": "business_finops", + "severity": [ + "P0", + "P1", + "P2", + "P3" + ], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "awoooi.alert_operation_log", + "owner": "finops_runtime", + "egress_status": "blocked", + "blocked_reason": "business_and_cost_signals_require_a_separate_owned_destination" + }, + { + "route_id": "bitan-product-health", + "product_id": "bitan-pharmacy", + "signal_family": "container_and_product_health", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "unassigned", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "bitan_product_ops", + "egress_status": "blocked", + "blocked_reason": "current_shared_sre_fanout_is_not_a_canonical_product_route" + }, + { + "route_id": "clawbot-raw-monitoring", + "product_id": "clawbot-openclaw", + "signal_family": "raw_monitoring_alert", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "tsenyang_bot", + "chat_alias": "tsenyangbot_private", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "ai_agent_runtime", + "egress_status": "blocked", + "blocked_reason": "cross_product_bot_ownership_drift_requires_a_clawbot_owned_binding" + }, + { + "route_id": "momo-data-freshness", + "product_id": "momo-pro", + "signal_family": "crawler_and_data_freshness", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "momo_bot", + "chat_alias": "momo_configured_chat_unresolved", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "momo_product_ops", + "egress_status": "blocked", + "blocked_reason": "configured_destination_returns_chat_not_found_and_has_no_durable_receipt" + }, + { + "route_id": "momo-scheduler-sales", + "product_id": "momo-pro", + "signal_family": "scheduler_and_sales_pipeline", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "momo_bot", + "chat_alias": "momo_configured_chat_unresolved", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "momo_product_ops", + "egress_status": "blocked", + "blocked_reason": "configured_destination_returns_chat_not_found_and_has_no_durable_receipt" + }, + { + "route_id": "stockplatform-market-data", + "product_id": "stockplatform-v2", + "signal_family": "market_data_and_recommendation_freshness", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "wooowooowooobot", + "chat_alias": "wooowooowooobot_private", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "stockplatform_product_ops", + "egress_status": "blocked", + "blocked_reason": "private_route_is_shared_with_tsenyang_site_and_product_ownership_is_not_reconciled" + }, + { + "route_id": "tsenyang-site-health", + "product_id": "tsenyang-website", + "signal_family": "public_site_and_agent_action_health", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "wooowooowooobot", + "chat_alias": "wooowooowooobot_private", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "tsenyang_site_ops", + "egress_status": "blocked", + "blocked_reason": "private_route_is_shared_with_stockplatform_and_product_ownership_is_not_reconciled" + }, + { + "route_id": "vibework-not-implemented", + "product_id": "vibework", + "signal_family": "*", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "vibework_product_ops", + "egress_status": "not_implemented", + "blocked_reason": "standalone_vibework_has_no_telegram_egress_agent_bounty_is_separate" + }, + { + "route_id": "vtuber-disabled", + "product_id": "vtuber", + "signal_family": "*", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "vtuber_product_ops", + "egress_status": "disabled", + "blocked_reason": "telegram_egress_disabled" + } + ], + "rollups": { + "product_count": 11, + "route_count": 20, + "allowed_route_count": 4, + "blocked_route_count": 13, + "disabled_route_count": 2, + "not_implemented_route_count": 1, + "shared_sre_allowed_route_count": 4, + "raw_numeric_destination_count": 0, + "runtime_route_change_count": 0, + "telegram_send_count": 0 + }, + "execution_boundaries": { + "telegram_send_allowed": false, + "bot_api_call_allowed": false, + "secret_value_read_allowed": false, + "raw_chat_id_storage_allowed": false, + "runtime_route_change_allowed": false, + "production_deploy_allowed": false + } +} diff --git a/scripts/reboot-recovery/awoooi-telegram-notify.sh b/scripts/reboot-recovery/awoooi-telegram-notify.sh index 4eee1609e..3646f0351 100755 --- a/scripts/reboot-recovery/awoooi-telegram-notify.sh +++ b/scripts/reboot-recovery/awoooi-telegram-notify.sh @@ -16,14 +16,19 @@ while [[ $# -gt 0 ]]; do esac done -: "${TELEGRAM_BOT_TOKEN:?TELEGRAM_BOT_TOKEN is required in host-local env}" -: "${TELEGRAM_CHAT_ID:?TELEGRAM_CHAT_ID is required in host-local env}" - TEXT="[AWOOOI reboot-recovery] phase=${PHASE} status=${STATUS} ${MESSAGE} artifact=${ARTIFACT_DIR}" -curl --fail --silent --show-error --max-time 10 \ - --request POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ - --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \ - --data-urlencode "text=${TEXT}" >/dev/null +# Reboot recovery is outside the canonical API receipt boundary. Do not read +# a host-local Telegram binding and do not fall back to a direct provider call. +# The orchestrator keeps its local evidence; a correlated gateway transport +# must be supplied before Telegram delivery can be re-enabled. +printf '%s\n' \ + "telegram_route_status=blocked_no_egress" \ + "telegram_provider_send_performed=false" \ + "telegram_blocked_reason=canonical_telegram_gateway_transport_required" \ + "telegram_message_class=reboot_recovery" \ + "telegram_phase=${PHASE}" \ + "telegram_status=${STATUS}" >&2 +exit 75 diff --git a/scripts/security/telegram-notification-egress-inventory.py b/scripts/security/telegram-notification-egress-inventory.py index 6ee829cbf..ec392c273 100644 --- a/scripts/security/telegram-notification-egress-inventory.py +++ b/scripts/security/telegram-notification-egress-inventory.py @@ -22,21 +22,37 @@ from typing import Any TAIPEI = timezone(timedelta(hours=8)) SCAN_ROOTS = ( + Path("agent99-control-plane.ps1"), Path(".gitea/workflows"), Path("scripts/ops"), Path("scripts/ci"), + Path("scripts/reboot-recovery"), Path("apps/api/src"), ) -SCAN_SUFFIXES = {".py", ".sh", ".js", ".yml", ".yaml"} +SCAN_SUFFIXES = {".py", ".ps1", ".sh", ".js", ".yml", ".yaml"} +GUARDED_BOT_METHODS = ( + "sendMessage", + "sendDocument", + "sendPhoto", + "sendMediaGroup", + "editMessageText", + "sendAnimation", + "sendVideo", + "sendAudio", + "sendVoice", +) DIRECT_BOT_API_RE = re.compile( - r"api\.telegram\.org/bot.*sendMessage|sendMessage.*api\.telegram\.org/bot" + r"api\.telegram\.org/bot.*?/(?P" + + "|".join(re.escape(method) for method in GUARDED_BOT_METHODS) + + r")\b", + re.IGNORECASE, ) GATEWAY_CALLSITE_RE = re.compile( r"(?:send_alert_notification\(|\b(?:tg|gw|gateway|telegram)\.send_text\(|_send_request\(\s*[\"']sendMessage[\"'])" ) SECRET_INTERPOLATION_RE = re.compile(r"\$\{\{\s*secrets\.[^}]+\}\}") -BOT_TOKEN_URL_RE = re.compile(r"api\.telegram\.org/bot.*?/sendMessage") +BOT_TOKEN_URL_RE = DIRECT_BOT_API_RE REQUIRED_OWNER_FIELDS = [ "egress_surface_id", @@ -134,6 +150,10 @@ def iter_scannable_files(root: Path) -> list[Path]: absolute_root = root / scan_root if not absolute_root.exists(): continue + if absolute_root.is_file(): + if absolute_root.suffix in SCAN_SUFFIXES: + files.append(absolute_root) + continue for path in absolute_root.rglob("*"): if path.is_file() and path.suffix in SCAN_SUFFIXES: files.append(path) @@ -143,7 +163,10 @@ def iter_scannable_files(root: Path) -> list[Path]: def sanitize_excerpt(line: str) -> str: excerpt = line.strip() excerpt = SECRET_INTERPOLATION_RE.sub("${{ secrets. }}", excerpt) - excerpt = BOT_TOKEN_URL_RE.sub("api.telegram.org/bot/sendMessage", excerpt) + excerpt = BOT_TOKEN_URL_RE.sub( + lambda match: f"api.telegram.org/bot/{match.group('method')}", + excerpt, + ) return excerpt[:180] @@ -154,6 +177,10 @@ def surface_kind(relative_path: str) -> str: return "ops_script_direct_bot_api" if relative_path.startswith("scripts/ci/"): return "ci_script_direct_bot_api" + if relative_path == "agent99-control-plane.ps1": + return "agent99_control_plane_direct_bot_api" + if relative_path.startswith("scripts/reboot-recovery/"): + return "reboot_recovery_direct_bot_api" if relative_path.startswith("apps/api/src/"): return "api_direct_bot_api" return "other_direct_bot_api" @@ -174,7 +201,8 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: relative_path = path.relative_to(root).as_posix() text = path.read_text(encoding="utf-8", errors="replace") for line_number, line in enumerate(text.splitlines(), start=1): - if DIRECT_BOT_API_RE.search(line): + direct_match = DIRECT_BOT_API_RE.search(line) + if direct_match: kind = surface_kind(relative_path) direct_calls.append( { @@ -183,6 +211,7 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: "path": relative_path, "line": line_number, "line_hash": line_hash(relative_path, line_number, line), + "method": direct_match.group("method"), "sanitized_excerpt": sanitize_excerpt(line), "required_owner_fields": REQUIRED_OWNER_FIELDS, "reviewer_checks": REVIEWER_CHECKS, diff --git a/scripts/security/telegram-notification-egress-no-new-bypass-guard.py b/scripts/security/telegram-notification-egress-no-new-bypass-guard.py index 0a45301c4..bc2ebc4bc 100644 --- a/scripts/security/telegram-notification-egress-no-new-bypass-guard.py +++ b/scripts/security/telegram-notification-egress-no-new-bypass-guard.py @@ -24,12 +24,14 @@ TAIPEI = timezone(timedelta(hours=8)) SOURCE_SNAPSHOT = Path("docs/security/telegram-notification-egress-inventory.snapshot.json") SCAN_ROOTS = ( + Path("agent99-control-plane.ps1"), Path(".gitea/workflows"), Path("scripts/ops"), Path("scripts/ci"), + Path("scripts/reboot-recovery"), Path("apps/api/src"), ) -SCAN_SUFFIXES = {".py", ".sh", ".js", ".yml", ".yaml"} +SCAN_SUFFIXES = {".py", ".ps1", ".sh", ".js", ".yml", ".yaml"} GUARDED_BOT_METHODS = ( "sendMessage", "sendDocument", @@ -77,6 +79,10 @@ def iter_scannable_files(root: Path) -> list[Path]: absolute_root = root / scan_root if not absolute_root.exists(): continue + if absolute_root.is_file(): + if absolute_root.suffix in SCAN_SUFFIXES: + files.append(absolute_root) + continue for path in absolute_root.rglob("*"): if path.is_file() and path.suffix in SCAN_SUFFIXES: files.append(path) From 849f06ad115ba81dbeca8a0ed14c60da840d1bf8 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 20:49:34 +0800 Subject: [PATCH 5/7] fix(telegram): fail closed on unverified delivery --- agent99-control-plane.ps1 | 340 +-------- .../src/services/notifications/telegram.py | 7 +- apps/api/src/services/telegram_gateway.py | 436 +++++++++--- ...nt99_transport_recovery_deploy_contract.py | 12 +- apps/api/tests/test_heartbeat_dedup_p0_4.py | 33 +- .../test_telegram_callback_context_routing.py | 117 +-- .../test_telegram_canonical_sender_gate.py | 334 ++++++++- .../test_telegram_delivery_truth_callers.py | 27 +- ...test_telegram_delivery_truth_callers_v2.py | 27 +- ...test_telegram_delivery_truth_gateway_v2.py | 115 ++- .../tests/test_telegram_message_templates.py | 74 +- ...t_telegram_notification_egress_scanners.py | 495 +++++++++++++ .../telegram-alert-readability-guard.py | 44 +- .../telegram-notification-egress-inventory.py | 665 ++++++++++++++++-- ...notification-egress-no-new-bypass-guard.py | 622 +++++++++++++++- 15 files changed, 2751 insertions(+), 597 deletions(-) create mode 100644 apps/api/tests/test_telegram_notification_egress_scanners.py diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index bfd210e6e..e1fc23f66 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -534,185 +534,6 @@ function Invoke-AgentStaleSshProcessGuard { $result } -function Send-AgentTelegramRelay { - param( - [string]$Message, - [object]$Relay, - [string]$ChatIdOverride = "", - [string]$PhotoPath = "", - [string]$ReplyToMessageId = "" - ) - - if (-not ($Relay -and $Relay.enabled)) { - return [pscustomobject]@{ ok = $false; error = "relay_disabled" } - } - - # Canonical Telegram routing is owned by the AWOOOI API gateway. Agent99 - # must never borrow a container token/chat binding or call Bot API directly. - # Until a correlated gateway transport is configured this path is an - # explicit no-egress terminal, not a best-effort fallback. - return [pscustomobject]@{ - ok = $false - sent = $false - providerSendPerformed = $false - routeStatus = "blocked_no_egress" - error = "canonical_telegram_gateway_transport_required" - } - - $relayHost = if ($Relay.host) { [string]$Relay.host } else { "192.168.0.110" } - $container = if ($Relay.container) { [string]$Relay.container } else { "stockplatform-v2-api-1" } - if ($container -notmatch "^[A-Za-z0-9_.-]+$") { - return [pscustomobject]@{ ok = $false; error = "invalid_relay_container" } - } - - $messageBytes = [Text.Encoding]::UTF8.GetBytes($Message) - $messageB64 = [Convert]::ToBase64String($messageBytes) - $chatOverrideB64 = if ($ChatIdOverride) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($ChatIdOverride)) } else { "" } - $photoSent = $false - $photoError = $null - $containerPhotoPath = "" - if ($PhotoPath -and (Test-Path -LiteralPath $PhotoPath)) { - $safePhotoName = "agent99-card-" + ([guid]::NewGuid().ToString("N")) + ".png" - $remotePhotoPath = "/tmp/" + $safePhotoName - $containerPhotoPath = "/tmp/" + $safePhotoName - $relayUser = Get-HostSshUser $relayHost - $scpArgs = @(Get-AgentSshIdentityArguments) + @( - "-o", "BatchMode=yes", - "-o", "StrictHostKeyChecking=accept-new", - "-o", "NumberOfPasswordPrompts=0", - "-o", "ConnectTimeout=10", - $PhotoPath, - "${relayUser}@${relayHost}:$remotePhotoPath" - ) - try { - $scp = Start-Process -FilePath "scp.exe" -ArgumentList $scpArgs -NoNewWindow -Wait -PassThru - $scp.Refresh() - if ($null -ne $scp.ExitCode -and $scp.ExitCode -eq 0) { - $copyRun = Invoke-SshText $relayHost "docker cp $remotePhotoPath $container`:$containerPhotoPath && rm -f $remotePhotoPath" 45 1 - if (-not $copyRun.ok) { - $photoError = "relay_docker_cp_failed: $($copyRun.output)" - $containerPhotoPath = "" - } - } else { - $photoError = "relay_scp_failed exitCode=$($scp.ExitCode)" - $containerPhotoPath = "" - } - } catch { - $photoError = "relay_scp_exception: $($_.Exception.Message)" - $containerPhotoPath = "" - } - } - - $containerPhotoB64 = if ($containerPhotoPath) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($containerPhotoPath)) } else { "" } - $python = 'import base64, json, mimetypes, os, sys, urllib.error, urllib.parse, urllib.request -text = base64.b64decode(sys.argv[2]).decode() -chat_override = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] and sys.argv[3] != "-" else "" -photo_path = base64.b64decode(sys.argv[4]).decode() if len(sys.argv) > 4 and sys.argv[4] and sys.argv[4] != "-" else "" -reply_to = int(sys.argv[5]) if len(sys.argv) > 5 and sys.argv[5].isdigit() else None -token = os.environ.get("TELEGRAM_BOT_TOKEN") -chat = chat_override or os.environ.get("TELEGRAM_CHAT_ID") -assert token and chat -reply_parameters = json.dumps({"message_id": reply_to}) if reply_to else None -def emit_receipt(payload): - result = payload.get("result") or {} - safe = {"message_id": result.get("message_id"), "chat_type": (result.get("chat") or {}).get("type")} - print("telegram_receipt_b64=" + base64.b64encode(json.dumps(safe).encode()).decode()) -if photo_path and os.path.exists(photo_path): - caption = text - if len(caption) > 900: - caption = caption[:900] + "\n...(truncated; see evidence)" - filename = os.path.basename(photo_path) - ctype = mimetypes.guess_type(filename)[0] or "image/png" - try: - import requests - request_data = {"chat_id": chat, "caption": caption} - if reply_parameters: - request_data["reply_parameters"] = reply_parameters - with open(photo_path, "rb") as fh: - response = requests.post( - "blocked://canonical-telegram-gateway-required/sendPhoto", - data=request_data, - files={"photo": (filename, fh, ctype)}, - timeout=30, - ) - if not response.ok: - print("photo_send_failed status=%s body=%s" % (response.status_code, response.text[:500])) - sys.exit(2) - emit_receipt(response.json()) - print("photo_sent_ok") - except ImportError: - boundary = "----agent99boundary" - fields = [("chat_id", chat), ("caption", caption)] - if reply_parameters: - fields.append(("reply_parameters", reply_parameters)) - body = bytearray() - for name, value in fields: - body.extend(("--" + boundary + "\r\n").encode()) - body.extend(("Content-Disposition: form-data; name=\"" + name + "\"\r\n\r\n").encode()) - body.extend(str(value).encode()) - body.extend(b"\r\n") - body.extend(("--" + boundary + "\r\n").encode()) - body.extend(("Content-Disposition: form-data; name=\"photo\"; filename=\"" + filename + "\"\r\n").encode()) - body.extend(("Content-Type: " + ctype + "\r\n\r\n").encode()) - with open(photo_path, "rb") as fh: - body.extend(fh.read()) - body.extend(b"\r\n") - body.extend(("--" + boundary + "--\r\n").encode()) - req = urllib.request.Request("blocked://canonical-telegram-gateway-required/sendPhoto", data=bytes(body)) - req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary) - try: - response_payload = json.loads(urllib.request.urlopen(req, timeout=30).read().decode()) - emit_receipt(response_payload) - print("photo_sent_ok") - except urllib.error.HTTPError as exc: - print("photo_send_failed status=%s body=%s" % (exc.code, exc.read().decode(errors="replace")[:500])) - sys.exit(2) -else: - fields = {"chat_id": chat, "text": text, "disable_web_page_preview": "true"} - if reply_parameters: - fields["reply_parameters"] = reply_parameters - data = urllib.parse.urlencode(fields).encode() - response_payload = json.loads(urllib.request.urlopen("blocked://canonical-telegram-gateway-required/sendMessage", data=data, timeout=15).read().decode()) - emit_receipt(response_payload) - print("sent_ok") -' - $pythonB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($python)) - $bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))' - $chatOverrideArg = if ($chatOverrideB64) { $chatOverrideB64 } else { "-" } - $containerPhotoArg = if ($containerPhotoB64) { $containerPhotoB64 } else { "-" } - $replyArg = if ($ReplyToMessageId -match "^\d+$") { $ReplyToMessageId } else { "-" } - $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideArg $containerPhotoArg $replyArg" - $run = Invoke-SshText $relayHost $remoteCommand 45 1 - $photoSent = [bool]($run.ok -and $run.output -match "photo_sent_ok") - $messageId = $null - $receiptMatch = [regex]::Match([string]$run.output, "telegram_receipt_b64=([A-Za-z0-9+/=]+)") - if ($receiptMatch.Success) { - try { - $receiptJson = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($receiptMatch.Groups[1].Value)) | ConvertFrom-Json - $messageId = $receiptJson.message_id - } catch { - $messageId = $null - } - } - if ($containerPhotoPath) { - Invoke-SshText $relayHost "docker exec $container rm -f $containerPhotoPath" 20 1 | Out-Null - } - - [pscustomobject]@{ - ok = [bool]($run.ok -and ($run.output -match "sent_ok|photo_sent_ok")) - host = $relayHost - container = $container - exitCode = $run.exitCode - chatOverride = if ($ChatIdOverride) { "telegram-origin" } else { $null } - replyToMessageId = if ($ReplyToMessageId) { $ReplyToMessageId } else { $null } - messageId = $messageId - photoSent = $photoSent - photoError = $photoError - output = $run.output - error = if ($run.ok) { $photoError } else { $run.output } - } -} - function Get-AgentObjectValue { param( [object]$Data, @@ -1644,60 +1465,6 @@ function New-AgentTelegramCardImage { } } -function Send-AgentTelegramPhotoDirect { - param( - [string]$Token, - [string]$ChatId, - [string]$Caption, - [string]$PhotoPath, - [string]$ReplyToMessageId = "" - ) - - return [pscustomobject]@{ - ok = $false - sent = $false - providerSendPerformed = $false - routeStatus = "blocked_no_egress" - error = "canonical_telegram_gateway_transport_required" - } - - Add-Type -AssemblyName System.Net.Http -ErrorAction Stop - $client = New-Object System.Net.Http.HttpClient - $content = New-Object System.Net.Http.MultipartFormDataContent - $fileContent = $null - try { - $content.Add([System.Net.Http.StringContent]::new($ChatId), "chat_id") - $captionToSend = $Caption - if ($captionToSend.Length -gt 900) { - $captionToSend = $captionToSend.Substring(0, 900) + "`n...(truncated; see evidence)" - } - $content.Add([System.Net.Http.StringContent]::new($captionToSend), "caption") - if ($ReplyToMessageId -match "^\d+$") { - $replyJson = @{ message_id = [long]$ReplyToMessageId } | ConvertTo-Json -Compress - $content.Add([System.Net.Http.StringContent]::new($replyJson), "reply_parameters") - } - $fileBytes = [IO.File]::ReadAllBytes($PhotoPath) - $fileContent = [System.Net.Http.ByteArrayContent]::new($fileBytes) - $fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("image/png") - $content.Add($fileContent, "photo", [IO.Path]::GetFileName($PhotoPath)) - $response = $client.PostAsync("blocked://canonical-telegram-gateway-required/sendPhoto", $content).GetAwaiter().GetResult() - $body = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() - if (-not $response.IsSuccessStatusCode) { - throw "telegram_sendPhoto_failed status=$([int]$response.StatusCode) body=$body" - } - $payload = $body | ConvertFrom-Json - return [pscustomobject]@{ - ok = [bool]$payload.ok - messageId = if ($payload.result) { $payload.result.message_id } else { $null } - replyToMessageId = if ($ReplyToMessageId) { $ReplyToMessageId } else { $null } - } - } finally { - if ($fileContent) { $fileContent.Dispose() } - if ($content) { $content.Dispose() } - if ($client) { $client.Dispose() } - } -} - function Invoke-AgentTelegramCardSelfTest { $sample = [pscustomobject]@{ host = "192.168.0.110" @@ -1940,33 +1707,6 @@ function Send-AgentTelegram { $telegram = $Config.telegram $enabled = ($telegram -and $telegram.enabled) - $botTokenEnv = if ($telegram.botTokenEnv) { $telegram.botTokenEnv } else { "AGENT99_TELEGRAM_BOT_TOKEN" } - $chatIdEnv = if ($telegram.chatIdEnv) { $telegram.chatIdEnv } else { "AGENT99_TELEGRAM_CHAT_ID" } - $tokenEnvAliases = @($botTokenEnv) - if ($telegram.botTokenEnvAliases) { $tokenEnvAliases += @($telegram.botTokenEnvAliases) } - $chatIdEnvAliases = @($chatIdEnv) - if ($telegram.chatIdEnvAliases) { $chatIdEnvAliases += @($telegram.chatIdEnvAliases) } - - function Get-AgentEnvSecret { - param([string[]]$Names) - foreach ($name in ($Names | Where-Object { $_ } | Select-Object -Unique)) { - foreach ($scope in @("Machine", "User", "Process")) { - $value = [Environment]::GetEnvironmentVariable($name, $scope) - if ($value) { - return [pscustomobject]@{ value = $value; name = $name; scope = $scope } - } - } - } - return $null - } - - # The canonical gateway transport is not wired into Agent99 yet. Keep all - # credential bindings unset so the fail-closed receipt below cannot read a - # host-local bot token/chat identifier before returning. - $tokenSecret = $null - $chatSecret = $null - $token = $null - $chatId = $null $chatIdOverride = if ($Data -and $Data.PSObject.Properties["replyChatId"]) { [string]$Data.replyChatId } else { "" } $card = Get-AgentIncidentCardModel $Severity $EventType $Message $Data $incidentState = Get-AgentTelegramIncidentState $card @@ -1984,9 +1724,9 @@ function Send-AgentTelegram { fingerprint = $card.fingerprint lifecycle = $card.lifecycle stateKey = $card.stateKey - configured = [bool]($token -and $chatId) - tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null } - chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null } + configured = $false + tokenEnv = $null + chatIdEnv = $null chatOverride = if ($chatIdOverride) { "telegram-origin" } else { $null } replyToMessageId = if ($replyToMessageId) { $replyToMessageId } else { $null } messageId = $null @@ -2019,80 +1759,6 @@ function Send-AgentTelegram { } $script:TelegramAttempts += $attempt return $attempt - - $text = Format-AgentTelegramText $Severity $EventType $Message $Data - $caption = Format-AgentTelegramCaption $Severity $EventType $Message $Data - $visualPath = Get-AgentTelegramVisualPath $Data - if (-not $visualPath) { - $visualPath = New-AgentTelegramCardImage $Severity $EventType $text $Data - } - $attempt.visualPath = $visualPath - if (-not ($token -and ($chatId -or $chatIdOverride))) { - $relayText = if ($visualPath) { $caption } else { $text } - $relayResult = Send-AgentTelegramRelay $relayText $telegram.relay $chatIdOverride $visualPath $replyToMessageId - $attempt.relay = $relayResult - if ($relayResult.ok) { - $attempt.sent = $true - $attempt.messageId = $relayResult.messageId - $attempt.error = $null - $attempt.visualSent = [bool]$relayResult.photoSent - if ($visualPath -and -not $relayResult.photoSent) { - $attempt.visualError = if ($relayResult.photoError) { $relayResult.photoError } else { "relay_sendPhoto_not_available" } - } - } else { - $attempt.error = "telegram_not_configured" - } - if ($attempt.sent) { - $attempt.incidentStatePath = Save-AgentTelegramIncidentState $card $incidentState $attempt.messageId $replyToMessageId - $attempt.incidentStateWritten = $true - } - $script:TelegramAttempts += $attempt - return $attempt - } - - try { - $targetChat = if ($chatIdOverride) { $chatIdOverride } else { $chatId } - if ($visualPath -and (Test-Path -LiteralPath $visualPath)) { - try { - $photoResult = Send-AgentTelegramPhotoDirect $token $targetChat $caption $visualPath $replyToMessageId - $attempt.sent = $true - $attempt.visualSent = $true - $attempt.messageId = $photoResult.messageId - } catch { - $attempt.visualError = $_.Exception.Message - Write-AgentLog "telegram_sendPhoto_failed event=$EventType fallback=sendMessage error=$($_.Exception.Message)" - $fallbackBody = @{ - chat_id = $targetChat - text = $text - disable_web_page_preview = "true" - } - if ($replyToMessageId) { $fallbackBody.reply_parameters = (@{ message_id = [long]$replyToMessageId } | ConvertTo-Json -Compress) } - $fallbackResult = Invoke-RestMethod -Method Post -Uri "blocked://canonical-telegram-gateway-required/sendMessage" -Body $fallbackBody - $attempt.sent = $true - $attempt.messageId = if ($fallbackResult.result) { $fallbackResult.result.message_id } else { $null } - } - } else { - $textBody = @{ - chat_id = $targetChat - text = $text - disable_web_page_preview = "true" - } - if ($replyToMessageId) { $textBody.reply_parameters = (@{ message_id = [long]$replyToMessageId } | ConvertTo-Json -Compress) } - $textResult = Invoke-RestMethod -Method Post -Uri "blocked://canonical-telegram-gateway-required/sendMessage" -Body $textBody - $attempt.sent = $true - $attempt.messageId = if ($textResult.result) { $textResult.result.message_id } else { $null } - } - } catch { - $attempt.error = $_.Exception.Message - } - - if ($attempt.sent) { - $attempt.incidentStatePath = Save-AgentTelegramIncidentState $card $incidentState $attempt.messageId $replyToMessageId - $attempt.incidentStateWritten = $true - } - - $script:TelegramAttempts += $attempt - return $attempt } function Record-AgentEvent { diff --git a/apps/api/src/services/notifications/telegram.py b/apps/api/src/services/notifications/telegram.py index 5eb7bb94b..52d865199 100644 --- a/apps/api/src/services/notifications/telegram.py +++ b/apps/api/src/services/notifications/telegram.py @@ -123,7 +123,10 @@ class TelegramWebhookProvider(NotificationProvider): if not self.enabled: return False try: - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gw = get_telegram_gateway() result = await gw.send_alert_notification( @@ -132,7 +135,7 @@ class TelegramWebhookProvider(NotificationProvider): signal_family="product_raw_monitoring", severity="P3", ) - return bool(result.get("ok")) + return _telegram_send_delivery_succeeded(result) except Exception as e: logger.error("telegram_connection_test_failed", error=str(e)) return False diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 00e382f00..25bc95342 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -28,6 +28,7 @@ import html import json import os import re +import secrets from collections.abc import Mapping from dataclasses import dataclass, field from datetime import UTC, datetime @@ -146,7 +147,26 @@ _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY = "_awooop_source_envelope_extra" _CANONICAL_ROUTE_CONTEXT_KEY = "_awoooi_canonical_telegram_route" _LEGACY_NOTIFICATION_TYPE_KEY = "_awoooi_legacy_notification_type" _NON_MONITORING_INTERACTION_KEY = "_awoooi_non_monitoring_interaction" +_BOT_TOKEN_OVERRIDE_KEY = "_awoooi_bot_token_override" +_ACTUAL_BOT_ALIAS_KEY = "_awoooi_actual_bot_alias" +_DELIVERY_CONTEXT_KEY = "_awoooi_delivery_context" _CANONICAL_SRE_DESTINATION = "telegram_chat_alias:awoooi_sre_war_room" +_GUARDED_TELEGRAM_BOT_METHODS = frozenset( + { + "sendMessage", + "sendDocument", + "sendPhoto", + "sendMediaGroup", + "editMessageText", + "sendAnimation", + "sendVideo", + "sendAudio", + "sendVoice", + } +) +_GUARDED_TELEGRAM_REPLY_METHODS = _GUARDED_TELEGRAM_BOT_METHODS - { + "editMessageText" +} _NON_MONITORING_INTERACTION_KINDS = frozenset( { "operator_command_reply", @@ -154,6 +174,64 @@ _NON_MONITORING_INTERACTION_KINDS = frozenset( "bot_discussion_reply", } ) + + +def _telegram_destination_binding(chat_id: object) -> str: + """Return a durable, redaction-safe binding for one Telegram destination.""" + if isinstance(chat_id, bool) or chat_id is None: + return "" + normalized = str(chat_id).strip() + if not normalized: + return "" + return hashlib.sha256( + f"telegram-destination-v1:{normalized}".encode() + ).hexdigest() + + +def _telegram_provider_delivery_evidence( + provider_result: object, +) -> tuple[list[int], str]: + """Extract positive message ids and one consistent provider chat binding.""" + if isinstance(provider_result, Mapping): + messages = [provider_result] + elif isinstance(provider_result, list) and provider_result: + messages = provider_result + else: + return [], "" + + message_ids: list[int] = [] + destination_bindings: set[str] = set() + destination_binding_complete = True + for message in messages: + if not isinstance(message, Mapping): + return [], "" + message_id = message.get("message_id") + if ( + isinstance(message_id, bool) + or not isinstance(message_id, int) + or message_id <= 0 + ): + return [], "" + provider_chat = message.get("chat") + if not isinstance(provider_chat, Mapping): + destination_binding_complete = False + message_ids.append(message_id) + continue + destination_binding = _telegram_destination_binding( + provider_chat.get("id") + ) + if not destination_binding: + destination_binding_complete = False + message_ids.append(message_id) + continue + message_ids.append(message_id) + destination_bindings.add(destination_binding) + + if not destination_binding_complete or len(destination_bindings) != 1: + return message_ids, "" + return message_ids, destination_bindings.pop() + + _CONTROLLED_APPLY_OUTBOUND_FINALIZE_ATTEMPTS = 3 _NO_WRITE_REPLAY_DIGEST_WINDOW_MINUTES = 30 _HOST_RESOURCE_ALERT_HEADER_RE = re.compile( @@ -853,15 +931,87 @@ def _telegram_send_delivery_succeeded(result: object) -> bool: return False delivery_status = str(result.get("_awooop_delivery_status") or "").strip() - if delivery_status and delivery_status not in {"sent", "sent_reused"}: + if delivery_status != "sent": return False - if result.get("_awooop_provider_send_performed") is False: + if result.get("_awooop_provider_send_performed") is not True: return False route_receipt = result.get("_awoooi_canonical_route_receipt") if ( - isinstance(route_receipt, Mapping) - and route_receipt.get("provider_send_performed") is False + not isinstance(route_receipt, Mapping) + or route_receipt.get("schema_version") + != "telegram_canonical_egress_receipt_v1" + or route_receipt.get("decision") != "allowed" + or route_receipt.get("provider_send_performed") is not True + ): + return False + + delivery_context = result.get(_DELIVERY_CONTEXT_KEY) + if ( + not isinstance(delivery_context, Mapping) + or delivery_context.get("schema_version") + != "telegram_delivery_context_v1" + or delivery_context.get("destination_binding_verified") is not True + ): + return False + + route_sender_bot_alias = str( + route_receipt.get("sender_bot_alias") or "" + ).strip() + final_exit_sender_bot_alias = str( + delivery_context.get("sender_bot_alias") or "" + ).strip() + if ( + not route_sender_bot_alias + or route_sender_bot_alias != final_exit_sender_bot_alias + ): + return False + + route_destination_alias = str( + route_receipt.get("destination_alias") or "" + ).strip() + final_exit_destination_alias = str( + delivery_context.get("destination_alias") or "" + ).strip() + if ( + not route_destination_alias + or route_destination_alias != final_exit_destination_alias + ): + return False + + route_destination_binding = str( + route_receipt.get("destination_binding") or "" + ).strip() + final_exit_destination_binding = str( + delivery_context.get("destination_binding") or "" + ).strip() + payload_destination_binding = str( + delivery_context.get("payload_destination_binding") or "" + ).strip() + provider_destination_binding = str( + delivery_context.get("provider_destination_binding") or "" + ).strip() + if ( + not route_destination_binding + or len( + { + route_destination_binding, + final_exit_destination_binding, + payload_destination_binding, + provider_destination_binding, + } + ) + != 1 + ): + return False + + provider_result = result.get("result") + provider_message_ids, observed_provider_binding = ( + _telegram_provider_delivery_evidence(provider_result) + ) + if ( + not provider_message_ids + or observed_provider_binding != provider_destination_binding ): return False return True @@ -5272,6 +5422,8 @@ class TelegramGateway: "signal_family": str(context.get("signal_family") or "unknown")[:80], "severity": str(context.get("severity") or "unknown")[:16], "destination_alias": "none", + "destination_binding": "", + "sender_bot_alias": "", "provider_send_performed": False, } @@ -5363,6 +5515,9 @@ class TelegramGateway: "classifier": "canonical_shared_route_allowed", **normalized_context, "destination_alias": "awoooi_sre_war_room", + "destination_binding": _telegram_destination_binding( + canonical_chat_id + ), "sender_bot_alias": actual_bot_alias, "provider_send_performed": False, } @@ -5370,6 +5525,7 @@ class TelegramGateway: def _authorize_non_monitoring_interaction( self, *, + method: str, payload: Mapping[str, object], interaction_context: object, actual_bot_alias: str, @@ -5412,10 +5568,18 @@ class TelegramGateway: route_context={"signal_family": interaction_kind}, ) - reply_parameters = payload.get("reply_parameters") - reply_message_raw = payload.get("reply_to_message_id") - if isinstance(reply_parameters, Mapping): - reply_message_raw = reply_parameters.get("message_id") + if method == "editMessageText": + reply_message_raw = payload.get("message_id") + elif method in _GUARDED_TELEGRAM_REPLY_METHODS: + reply_parameters = payload.get("reply_parameters") + reply_message_raw = payload.get("reply_to_message_id") + if isinstance(reply_parameters, Mapping): + reply_message_raw = reply_parameters.get("message_id") + else: + return None, self._blocked_route_receipt( + "interaction_method_not_guarded", + route_context={"signal_family": interaction_kind}, + ) try: reply_message_id = int(str(reply_message_raw)) except (TypeError, ValueError): @@ -5441,6 +5605,9 @@ class TelegramGateway: "signal_family": interaction_kind, "severity": "N/A", "destination_alias": "inbound_interaction_reply_target", + "destination_binding": _telegram_destination_binding( + target_chat_id + ), "sender_bot_alias": actual_bot_alias, "inbound_context_verified": True, "provider_send_performed": False, @@ -5973,6 +6140,38 @@ class TelegramGateway: dict: API 回應 """ payload = dict(payload) + actual_bot_alias = str( + payload.pop(_ACTUAL_BOT_ALIAS_KEY, "tsenyang_bot") + ).strip() + token_override = payload.pop(_BOT_TOKEN_OVERRIDE_KEY, None) + provider_bot_token = self.bot_token + if token_override is None: + bot_binding_valid = actual_bot_alias == "tsenyang_bot" + else: + expected_token = { + "openclaw_bot": settings.OPENCLAW_BOT_TOKEN, + "nemotron_bot": settings.NEMOTRON_BOT_TOKEN, + }.get(actual_bot_alias, "") + bot_binding_valid = bool( + isinstance(token_override, str) + and expected_token + and secrets.compare_digest(token_override, expected_token) + ) + if bot_binding_valid: + provider_bot_token = str(token_override) + + if not bot_binding_valid: + route_receipt = self._blocked_route_receipt( + "sender_bot_binding_mismatch" + ) + return { + "ok": False, + "_awooop_outbound_mirror_acknowledged": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": route_receipt, + } + route_context = payload.pop(_CANONICAL_ROUTE_CONTEXT_KEY, None) legacy_notification_type = payload.pop( _LEGACY_NOTIFICATION_TYPE_KEY, @@ -5983,22 +6182,34 @@ class TelegramGateway: None, ) canonical_route_receipt: dict[str, object] | None = None - if method == "sendMessage": - if route_context is not None or legacy_notification_type is not None: + if method in _GUARDED_TELEGRAM_BOT_METHODS: + has_route_context = ( + route_context is not None + or legacy_notification_type is not None + ) + if has_route_context and interaction_context is not None: + canonical_chat_id, canonical_route_receipt = ( + None, + self._blocked_route_receipt( + "ambiguous_route_and_interaction_context" + ), + ) + elif has_route_context: canonical_chat_id, canonical_route_receipt = ( self._authorize_canonical_send_message( payload=payload, route_context=route_context, legacy_notification_type=legacy_notification_type, - actual_bot_alias="tsenyang_bot", + actual_bot_alias=actual_bot_alias, ) ) elif interaction_context is not None: canonical_chat_id, canonical_route_receipt = ( self._authorize_non_monitoring_interaction( + method=method, payload=payload, interaction_context=interaction_context, - actual_bot_alias="tsenyang_bot", + actual_bot_alias=actual_bot_alias, ) ) else: @@ -6121,7 +6332,7 @@ class TelegramGateway: source_envelope_extra = dict(source_envelope_extra or {}) source_envelope_extra["agent99_dispatch_receipt"] = agent99_receipt - url = f"{self.api_url}/{method}" + url = f"{self.TELEGRAM_API_BASE}/bot{provider_bot_token}/{method}" # OTEL Span: telegram.api.{method} with _tracer.start_as_current_span( @@ -6144,12 +6355,80 @@ class TelegramGateway: f"Telegram API error: {result.get('description', 'Unknown error')}" ) - # 成功: 記錄 message_id (result 可能是 dict 或 bool，需防禦) + # Guarded methods reach terminal delivery only when Telegram's + # result binds the provider message back to the exact sender + # and destination authorized before this single final exit. result_val = result.get("result") - if isinstance(result_val, dict) and "message_id" in result_val: - span.set_attribute("telegram.message_id", result_val["message_id"]) - provider_message_id = str(result_val["message_id"]) - if ( + provider_message_ids, provider_destination_binding = ( + _telegram_provider_delivery_evidence(result_val) + ) + has_provider_message_id = bool(provider_message_ids) + provider_message_id_value = ( + provider_message_ids[0] + if provider_message_ids + else None + ) + payload_destination_binding = ( + _telegram_destination_binding(payload.get("chat_id")) + ) + route_destination_binding = str( + (canonical_route_receipt or {}).get( + "destination_binding" + ) + or "" + ) + destination_binding_verified = bool( + canonical_route_receipt is not None + and route_destination_binding + and route_destination_binding + == payload_destination_binding + == provider_destination_binding + ) + delivery_context = { + "schema_version": "telegram_delivery_context_v1", + "method": method, + "sender_bot_alias": actual_bot_alias, + "destination_alias": str( + (canonical_route_receipt or {}).get( + "destination_alias" + ) + or "" + ), + "destination_binding": route_destination_binding, + "payload_destination_binding": ( + payload_destination_binding + ), + "provider_destination_binding": ( + provider_destination_binding + ), + "destination_binding_verified": ( + destination_binding_verified + ), + } + result[_DELIVERY_CONTEXT_KEY] = delivery_context + if has_provider_message_id: + span.set_attribute( + "telegram.message_id", + provider_message_id_value, + ) + provider_message_id = str(provider_message_id_value) + if canonical_route_receipt is not None: + canonical_route_receipt["provider_send_performed"] = True + result["_awoooi_canonical_route_receipt"] = ( + canonical_route_receipt + ) + if not destination_binding_verified: + if controlled_reservation is not None: + result[ + "_awooop_outbound_mirror_acknowledged" + ] = False + result["_awooop_delivery_status"] = ( + "provider_destination_mismatch" + if provider_destination_binding + else "provider_destination_unverified" + ) + result["_awooop_provider_send_performed"] = True + elif ( controlled_reservation is not None and controlled_delivery_identity is not None ): @@ -6166,8 +6445,8 @@ class TelegramGateway: ) result["_awooop_provider_send_performed"] = True else: - if canonical_route_receipt is not None: - canonical_route_receipt["provider_send_performed"] = True + result["_awooop_delivery_status"] = "sent" + result["_awooop_provider_send_performed"] = True source_envelope_extra = ( _acknowledge_callback_reply_source_envelope( source_envelope_extra, @@ -6181,13 +6460,20 @@ class TelegramGateway: provider_message_id=provider_message_id, source_envelope_extra=source_envelope_extra, ) - elif controlled_reservation is not None: - result["_awooop_outbound_mirror_acknowledged"] = False - result["_awooop_delivery_status"] = "pending_unknown" + else: + if controlled_reservation is not None: + result["_awooop_outbound_mirror_acknowledged"] = False + if canonical_route_receipt is not None: + canonical_route_receipt["provider_send_performed"] = True + result["_awooop_delivery_status"] = ( + "provider_message_id_missing" + ) result["_awooop_provider_send_performed"] = True - if canonical_route_receipt is not None: - canonical_route_receipt["provider_send_performed"] = True + if ( + canonical_route_receipt is not None + and "_awoooi_canonical_route_receipt" not in result + ): result["_awoooi_canonical_route_receipt"] = ( canonical_route_receipt ) @@ -9139,6 +9425,11 @@ class TelegramGateway: "parse_mode": "HTML", "reply_markup": {"inline_keyboard": []}, "disable_web_page_preview": True, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": inbound_chat_id, + "inbound_message_id": inbound_message_id, + }, }) except Exception: # 移除按鈕保底 @@ -9777,6 +10068,11 @@ class TelegramGateway: "parse_mode": "HTML", "reply_markup": {"inline_keyboard": []}, "disable_web_page_preview": True, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": inbound_chat_id, + "inbound_message_id": inbound_message_id, + }, }) except TelegramGatewayError as e: # 文字更新失敗不影響整體流程，按鈕已移除 @@ -11553,8 +11849,7 @@ class TelegramGateway: actual_bot_alias: str, inbound_chat_id: str | int | None = None, ) -> dict: - """ - 用指定 Bot Token 發訊息。 + """Route an alternate owned bot through the canonical final exit. Args: token: Bot Token @@ -11565,82 +11860,31 @@ class TelegramGateway: Returns: dict: Telegram API 回應 """ - route_context = ( - { + payload: dict[str, object] = { + "text": text[:4096], + "parse_mode": parse_mode, + _BOT_TOKEN_OVERRIDE_KEY: token, + _ACTUAL_BOT_ALIAS_KEY: actual_bot_alias, + } + if product_id and signal_family and severity: + payload[_CANONICAL_ROUTE_CONTEXT_KEY] = { "product_id": product_id, "signal_family": signal_family, "severity": severity, } - if product_id and signal_family and severity - else None - ) - candidate_payload: dict[str, object] = { - "chat_id": str(inbound_chat_id or settings.SRE_GROUP_CHAT_ID or ""), - "text": text[:4096], - "parse_mode": parse_mode, - } - if reply_to_message_id: - candidate_payload["reply_parameters"] = { + elif reply_to_message_id: + payload["chat_id"] = str(inbound_chat_id or "") + payload[_NON_MONITORING_INTERACTION_KEY] = { + "interaction_kind": "bot_discussion_reply", + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": reply_to_message_id, + } + if reply_to_message_id is not None: + payload["reply_parameters"] = { "message_id": reply_to_message_id, "allow_sending_without_reply": True, } - - if route_context is not None: - chat_id, route_receipt = self._authorize_canonical_send_message( - payload=candidate_payload, - route_context=route_context, - legacy_notification_type=None, - actual_bot_alias=actual_bot_alias, - ) - elif reply_to_message_id: - chat_id, route_receipt = self._authorize_non_monitoring_interaction( - payload=candidate_payload, - interaction_context={ - "interaction_kind": "bot_discussion_reply", - "inbound_chat_id": str(inbound_chat_id or ""), - "inbound_message_id": reply_to_message_id, - }, - actual_bot_alias=actual_bot_alias, - ) - else: - chat_id, route_receipt = ( - None, - self._blocked_route_receipt( - "missing_product_signal_severity_or_interaction_context" - ), - ) - if chat_id is None: - return { - "ok": False, - "_awooop_outbound_mirror_acknowledged": False, - "_awooop_delivery_status": "blocked_no_egress", - "_awooop_provider_send_performed": False, - "_awoooi_canonical_route_receipt": route_receipt, - } - - if not self._initialized: - await self.initialize() - if not self._http_client: - raise TelegramGatewayError("HTTP client not initialized") - - url = f"{self.TELEGRAM_API_BASE}/bot{token}/sendMessage" - payload = dict(candidate_payload) - payload["chat_id"] = chat_id - - response = await self._http_client.post(url, json=payload) - response.raise_for_status() - result = response.json() - route_receipt["provider_send_performed"] = True - if isinstance(result, dict): - result["_awoooi_canonical_route_receipt"] = route_receipt - result_val = result.get("result") if isinstance(result, dict) else None - if isinstance(result_val, dict) and "message_id" in result_val: - await self._mirror_outbound_message( - method="sendMessage", - payload=payload, - provider_message_id=str(result_val["message_id"]), - ) - return result + return await self._send_request("sendMessage", payload) async def send_as_openclaw( self, diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 112cb8c9e..3ed3908d6 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -96,7 +96,7 @@ def test_agent99_v5_formatter_synthetic_gate_loads_model_dependencies() -> None: assert 'C:\\Wooo' in synthetic -def test_agent99_telegram_threads_source_alert_and_persists_lifecycle() -> None: +def test_agent99_telegram_lifecycle_fails_closed_without_direct_sender() -> None: source = CONTROL.read_text(encoding="utf-8") inbox = SRE_INBOX.read_text(encoding="utf-8") @@ -104,8 +104,6 @@ def test_agent99_telegram_threads_source_alert_and_persists_lifecycle() -> None: assert "replyMessageId = $replyMessageId" in inbox assert 'PSObject.Properties["replyMessageId"]' in source assert 'PSObject.Properties["correlationKey"]' in source - assert '"reply_parameters"' in source - assert 'telegram_receipt_b64=' in source assert 'schemaVersion = "agent99_telegram_incident_state_v1"' in source assert 'rootMessageId = $rootMessageId' in source assert 'reason = "same_lifecycle_state"' in source @@ -113,6 +111,12 @@ def test_agent99_telegram_threads_source_alert_and_persists_lifecycle() -> None: assert "lastRunKey = $Card.runKey" in source assert "$previousRunKey -eq [string]$incidentCard.runKey" in source assert '$dedupeParts += "run=$($incidentCard.runKey)"' in source + assert 'error = "canonical_telegram_gateway_transport_required"' in source + assert 'providerSendPerformed = $false' in source + assert 'routeStatus = "blocked_no_egress"' in source + assert "function Send-AgentTelegramRelay" not in source + assert "function Send-AgentTelegramPhotoDirect" not in source + assert "telegram_receipt_b64=" not in source def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None: @@ -495,7 +499,7 @@ def test_agent99_bounds_stale_ssh_processes_without_touching_tunnels() -> None: source = CONTROL.read_text(encoding="utf-8") config = json.loads((ROOT / "agent99.config.99.example.json").read_text()) guard = source[source.index("function Invoke-AgentStaleSshProcessGuard") :] - guard = guard[: guard.index("function Send-AgentTelegramRelay")] + guard = guard[: guard.index("function Get-AgentObjectValue")] assert 'Get-CimInstance Win32_Process -Filter "Name=\'ssh.exe\'"' in guard assert 'CommandLine -cmatch "(^|\\s)-[NLRD](\\s|$)"' in guard diff --git a/apps/api/tests/test_heartbeat_dedup_p0_4.py b/apps/api/tests/test_heartbeat_dedup_p0_4.py index 691c3cf88..6e3d8970a 100644 --- a/apps/api/tests/test_heartbeat_dedup_p0_4.py +++ b/apps/api/tests/test_heartbeat_dedup_p0_4.py @@ -14,6 +14,37 @@ from unittest.mock import AsyncMock, patch import pytest +import src.services.telegram_gateway as gateway_module + +_SENT_CHAT_ID = -1001 +_SENT_DESTINATION_BINDING = gateway_module._telegram_destination_binding( + _SENT_CHAT_ID +) +_SENT_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 42, "chat": {"id": _SENT_CHAT_ID}}, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "sender_bot_alias": "tsenyang_bot", + "provider_send_performed": True, + }, + gateway_module._DELIVERY_CONTEXT_KEY: { + "schema_version": "telegram_delivery_context_v1", + "method": "sendMessage", + "sender_bot_alias": "tsenyang_bot", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "payload_destination_binding": _SENT_DESTINATION_BINDING, + "provider_destination_binding": _SENT_DESTINATION_BINDING, + "destination_binding_verified": True, + }, +} + class FakeRedis: """模擬 Redis 行為，記錄 set/get/delete 呼叫""" @@ -78,7 +109,7 @@ def gateway_with_fake_redis(): gw = TelegramGateway.__new__(TelegramGateway) # 跳過 __init__ gw._initialized = True gw._last_message_time = None - gw.send_to_group = AsyncMock() + gw.send_to_group = AsyncMock(return_value=dict(_SENT_RECEIPT)) gw.send_notification = AsyncMock() fake_redis = FakeRedis() diff --git a/apps/api/tests/test_telegram_callback_context_routing.py b/apps/api/tests/test_telegram_callback_context_routing.py index 3eca42751..5317eeaef 100644 --- a/apps/api/tests/test_telegram_callback_context_routing.py +++ b/apps/api/tests/test_telegram_callback_context_routing.py @@ -20,6 +20,31 @@ from src.services.telegram_gateway import TelegramGateway _CHAT_ID = "verified-chat" _MESSAGE_ID = 77 +_DESTINATION_BINDING = gateway_module._telegram_destination_binding(_CHAT_ID) +_SENT_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": _MESSAGE_ID, "chat": {"id": _CHAT_ID}}, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "destination_alias": "inbound_interaction_reply_target", + "destination_binding": _DESTINATION_BINDING, + "sender_bot_alias": "tsenyang_bot", + "provider_send_performed": True, + }, + gateway_module._DELIVERY_CONTEXT_KEY: { + "schema_version": "telegram_delivery_context_v1", + "method": "sendMessage", + "sender_bot_alias": "tsenyang_bot", + "destination_alias": "inbound_interaction_reply_target", + "destination_binding": _DESTINATION_BINDING, + "payload_destination_binding": _DESTINATION_BINDING, + "provider_destination_binding": _DESTINATION_BINDING, + "destination_binding_verified": True, + }, +} def _assert_verified_callback_payload(payload: dict) -> None: @@ -174,13 +199,15 @@ async def test_llm_and_category_results_keep_verified_callback_context( class _Redis: async def get(self, _key: str) -> str: - return json.dumps({ - "name": "inspect", - "provider": "internal", - "tool": "status", - "risk": "low", - "incident_id": "INC-1", - }) + return json.dumps( + { + "name": "inspect", + "provider": "internal", + "tool": "status", + "risk": "low", + "incident_id": "INC-1", + } + ) monkeypatch.setattr(gateway_module, "get_redis", lambda: _Redis()) monkeypatch.setattr( @@ -213,13 +240,15 @@ async def test_llm_and_category_results_keep_verified_callback_context( monkeypatch.setattr( callback_dispatcher, "dispatch_action", - AsyncMock(return_value=DispatchResult( - success=True, - action="inspect", - incident_id="INC-1", - user_id=42, - result_text="result", - )), + AsyncMock( + return_value=DispatchResult( + success=True, + action="inspect", + incident_id="INC-1", + user_id=42, + result_text="result", + ) + ), ) class _IncidentRepo: @@ -264,7 +293,9 @@ async def test_write_category_callback_keeps_verified_context( "verify_callback", AsyncMock(return_value={"id": 42}), ) - monkeypatch.setattr(gateway, "_check_incident_state_guard", AsyncMock(return_value=None)) + monkeypatch.setattr( + gateway, "_check_incident_state_guard", AsyncMock(return_value=None) + ) monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) spec = SimpleNamespace( requires_multi_sig=False, @@ -278,13 +309,15 @@ async def test_write_category_callback_keeps_verified_context( monkeypatch.setattr( callback_dispatcher, "dispatch_action", - AsyncMock(return_value=DispatchResult( - success=True, - action="restart_safe", - incident_id="INC-1", - user_id=42, - result_text="done", - )), + AsyncMock( + return_value=DispatchResult( + success=True, + action="restart_safe", + incident_id="INC-1", + user_id=42, + result_text="done", + ) + ), ) class _IncidentRepo: @@ -335,9 +368,7 @@ async def test_approval_replies_keep_verified_callback_context( _assert_verified_callback_payload(status_payload) source_extra = status_payload[gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] merged = gateway_module._merge_outbound_source_envelope_extra({}, source_extra) - assert merged["callback_reply"]["parent_provider_message_id"] == str( - _MESSAGE_ID - ) + assert merged["callback_reply"]["parent_provider_message_id"] == str(_MESSAGE_ID) send_request.reset_mock() approval_id = UUID("44444444-4444-4444-4444-444444444444") @@ -367,8 +398,9 @@ async def test_approval_replies_keep_verified_callback_context( @pytest.mark.parametrize( ("result", "expected"), [ - ({"ok": True}, True), - ({"ok": True, "_awooop_delivery_status": "sent_reused"}, True), + (_SENT_RECEIPT, True), + ({"ok": True}, False), + ({"ok": True, "_awooop_delivery_status": "sent_reused"}, False), ({"ok": False}, False), ( {"ok": True, "_awooop_delivery_status": "blocked_no_egress"}, @@ -414,7 +446,7 @@ async def test_approval_structured_no_send_retries_plain_on_exact_parent( ) -> None: gateway = TelegramGateway() attempts: list[tuple[str, dict]] = [] - send_results = iter([primary_failure, {"ok": True}]) + send_results = iter([primary_failure, dict(_SENT_RECEIPT)]) async def structured_primary_failure(method: str, payload: dict) -> dict: attempts.append((method, payload)) @@ -423,7 +455,7 @@ async def test_approval_structured_no_send_retries_plain_on_exact_parent( return {"ok": True} record_failure = AsyncMock(return_value=None) - shared_fallback = AsyncMock(return_value={"ok": True}) + shared_fallback = AsyncMock(return_value=dict(_SENT_RECEIPT)) monkeypatch.setattr(gateway, "_send_request", structured_primary_failure) monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) monkeypatch.setattr(gateway, "send_alert_notification", shared_fallback) @@ -454,10 +486,12 @@ async def test_approval_structured_double_no_send_records_durable_failure( ) -> None: gateway = TelegramGateway() attempts: list[tuple[str, dict]] = [] - send_results = iter([ - {"ok": True, "_awooop_delivery_status": "blocked_no_egress"}, - {"ok": True, "_awooop_provider_send_performed": False}, - ]) + send_results = iter( + [ + {"ok": True, "_awooop_delivery_status": "blocked_no_egress"}, + {"ok": True, "_awooop_provider_send_performed": False}, + ] + ) async def structured_no_send(method: str, payload: dict) -> dict: attempts.append((method, payload)) @@ -501,7 +535,7 @@ async def test_callback_card_edits_use_only_verified_inbound_identity( action: str, ) -> None: gateway = TelegramGateway() - send_request = AsyncMock(return_value={"ok": True}) + send_request = AsyncMock(return_value=dict(_SENT_RECEIPT)) monkeypatch.setattr(gateway, "_send_request", send_request) await gateway._update_message_after_action( @@ -658,17 +692,20 @@ async def test_autonomous_lifecycle_threads_use_canonical_route_and_parent_recei monkeypatch: pytest.MonkeyPatch, ) -> None: gateway = TelegramGateway() - send_request = AsyncMock(return_value={"ok": True}) + send_request = AsyncMock(return_value=dict(_SENT_RECEIPT)) monkeypatch.setattr(gateway, "_send_request", send_request) monkeypatch.setattr(gateway_module, "get_redis", lambda: _LifecycleRedis()) assert await gateway.mark_auto_repaired("INC-1", "playbook", 10) is True assert await gateway.append_incident_update("INC-2", "healthy") is True - assert await gateway.append_grouped_alert_digest( - incident_id="INC-3", - group_key="group-1", - digest_text="digest", - ) is True + assert ( + await gateway.append_grouped_alert_digest( + incident_id="INC-3", + group_key="group-1", + digest_text="digest", + ) + is True + ) send_payloads = [ call.args[1] diff --git a/apps/api/tests/test_telegram_canonical_sender_gate.py b/apps/api/tests/test_telegram_canonical_sender_gate.py index 2231e277a..02cc352f6 100644 --- a/apps/api/tests/test_telegram_canonical_sender_gate.py +++ b/apps/api/tests/test_telegram_canonical_sender_gate.py @@ -1,6 +1,8 @@ from __future__ import annotations +import runpy from contextlib import asynccontextmanager +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock @@ -11,13 +13,24 @@ import src.services.approval_db as approval_db import src.services.telegram_gateway as gateway_module from src.services.telegram_gateway import TelegramGateway +_REPO_ROOT = Path(__file__).resolve().parents[3] + class _FakeResponse: + def __init__(self, payload: dict) -> None: + self._payload = payload + def raise_for_status(self) -> None: return None def json(self) -> dict: - return {"ok": True, "result": {"message_id": 11}} + return { + "ok": True, + "result": { + "message_id": 11, + "chat": {"id": self._payload.get("chat_id")}, + }, + } class _FakeHttpClient: @@ -26,10 +39,12 @@ class _FakeHttpClient: async def post(self, url: str, json: dict) -> _FakeResponse: self.posts.append((url, json)) - return _FakeResponse() + return _FakeResponse(json) -def _prepared_gateway(monkeypatch: pytest.MonkeyPatch) -> tuple[TelegramGateway, _FakeHttpClient]: +def _prepared_gateway( + monkeypatch: pytest.MonkeyPatch, +) -> tuple[TelegramGateway, _FakeHttpClient]: gateway = TelegramGateway() client = _FakeHttpClient() gateway._initialized = True @@ -115,6 +130,77 @@ async def test_notification_provider_requires_affirmative_delivery_ack( assert result.error == "delivery_not_acknowledged" +@pytest.mark.asyncio +async def test_notification_connection_probe_rejects_structured_no_provider_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services.notifications import telegram as provider_module + + monkeypatch.setattr(provider_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(provider_module.settings, "SRE_GROUP_CHAT_ID", "test-room") + gateway = SimpleNamespace( + send_alert_notification=AsyncMock( + return_value={ + "ok": True, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": { + "decision": "blocked_no_egress", + "provider_send_performed": False, + }, + } + ) + ) + monkeypatch.setattr(gateway_module, "get_telegram_gateway", lambda: gateway) + + assert await provider_module.TelegramWebhookProvider().test_connection() is False + gateway.send_alert_notification.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_notification_connection_probe_accepts_complete_canonical_delivery( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services.notifications import telegram as provider_module + + monkeypatch.setattr(provider_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(provider_module.settings, "SRE_GROUP_CHAT_ID", "test-room") + chat_id = -1001 + destination_binding = gateway_module._telegram_destination_binding(chat_id) + gateway = SimpleNamespace( + send_alert_notification=AsyncMock( + return_value={ + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 42, "chat": {"id": chat_id}}, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": destination_binding, + "sender_bot_alias": "tsenyang_bot", + "provider_send_performed": True, + }, + gateway_module._DELIVERY_CONTEXT_KEY: { + "schema_version": "telegram_delivery_context_v1", + "method": "sendMessage", + "sender_bot_alias": "tsenyang_bot", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": destination_binding, + "payload_destination_binding": destination_binding, + "provider_destination_binding": destination_binding, + "destination_binding_verified": True, + }, + } + ) + ) + monkeypatch.setattr(gateway_module, "get_telegram_gateway", lambda: gateway) + + assert await provider_module.TelegramWebhookProvider().test_connection() is True + gateway.send_alert_notification.assert_awaited_once() + + @pytest.mark.asyncio async def test_unclassified_send_message_is_blocked_before_provider( monkeypatch: pytest.MonkeyPatch, @@ -131,6 +217,92 @@ async def test_unclassified_send_message_is_blocked_before_provider( assert client.posts == [] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method", + sorted(gateway_module._GUARDED_TELEGRAM_BOT_METHODS), +) +async def test_every_guarded_method_without_context_is_blocked_before_provider( + monkeypatch: pytest.MonkeyPatch, + method: str, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway._send_request( + method, + { + "chat_id": "test-room", + "text": "unclassified", + "caption": "unclassified", + "photo": "file-placeholder", + "document": "file-placeholder", + "media": [], + "message_id": 42, + }, + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awooop_provider_send_performed"] is False + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_guarded_photo_accepts_only_a_valid_canonical_route( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway._send_request( + "sendPhoto", + { + "chat_id": "test-room", + "photo": "file-placeholder", + gateway_module._CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "security_incident", + "severity": "P1", + }, + }, + ) + + assert result["_awooop_delivery_status"] == "sent" + assert gateway_module._telegram_send_delivery_succeeded(result) is True + assert len(client.posts) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("method", "method_payload"), + [ + ("sendDocument", {"document": "file-placeholder", "reply_to_message_id": 42}), + ("editMessageText", {"text": "updated", "message_id": 42}), + ], +) +async def test_guarded_interaction_method_requires_exact_inbound_binding( + monkeypatch: pytest.MonkeyPatch, + method: str, + method_payload: dict, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway._send_request( + method, + { + "chat_id": "operator-room", + **method_payload, + gateway_module._NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": "operator-room", + "inbound_message_id": 42, + }, + }, + ) + + assert result["_awooop_delivery_status"] == "sent" + assert gateway_module._telegram_send_delivery_succeeded(result) is True + assert len(client.posts) == 1 + + @pytest.mark.asyncio async def test_info_and_business_methods_are_blocked_before_provider( monkeypatch: pytest.MonkeyPatch, @@ -209,15 +381,77 @@ async def test_allowed_legacy_monitoring_methods_use_canonical_provider_once( assert result["ok"] is True assert result["_awoooi_canonical_route_receipt"]["decision"] == "allowed" assert result["_awoooi_canonical_route_receipt"]["provider_send_performed"] is True + assert gateway_module._telegram_send_delivery_succeeded(result) is True assert len(client.posts) == 1 assert client.posts[0][1]["chat_id"] == "test-room" +@pytest.mark.asyncio +async def test_provider_ok_without_message_id_is_non_terminal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + monkeypatch.setattr( + _FakeResponse, + "json", + lambda _self: {"ok": True, "result": {}}, + ) + + result = await gateway.send_escalation_card( + incident_id="INC-TEST", + original_alertname="Outage", + duration_min=15, + ) + + assert result["_awooop_delivery_status"] == "provider_message_id_missing" + assert result["_awooop_provider_send_performed"] is True + assert result["_awoooi_canonical_route_receipt"]["provider_send_performed"] is True + assert gateway_module._telegram_send_delivery_succeeded(result) is False + assert len(client.posts) == 1 + + +@pytest.mark.asyncio +async def test_provider_chat_mismatch_is_non_terminal_delivery( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + monkeypatch.setattr( + _FakeResponse, + "json", + lambda _self: { + "ok": True, + "result": {"message_id": 11, "chat": {"id": "wrong-room"}}, + }, + ) + + result = await gateway.send_escalation_card( + incident_id="INC-TEST", + original_alertname="Outage", + duration_min=15, + ) + + assert result["_awooop_delivery_status"] == "provider_destination_mismatch" + assert result["_awooop_provider_send_performed"] is True + assert ( + result[gateway_module._DELIVERY_CONTEXT_KEY][ + "destination_binding_verified" + ] + is False + ) + assert gateway_module._telegram_send_delivery_succeeded(result) is False + assert len(client.posts) == 1 + + @pytest.mark.asyncio async def test_openclaw_cannot_own_tsenyang_monitoring_route( monkeypatch: pytest.MonkeyPatch, ) -> None: gateway, client = _prepared_gateway(monkeypatch) + monkeypatch.setattr( + gateway_module.settings, + "OPENCLAW_BOT_TOKEN", + "test-token-placeholder", + ) result = await gateway._send_as_bot( token="test-token-placeholder", @@ -235,6 +469,100 @@ async def test_openclaw_cannot_own_tsenyang_monitoring_route( assert client.posts == [] +@pytest.mark.asyncio +async def test_alternate_bot_uses_only_canonical_send_request( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + canonical_send = AsyncMock( + return_value={ + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 11}, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "provider_send_performed": True, + }, + } + ) + monkeypatch.setattr(gateway, "_send_request", canonical_send) + + result = await gateway._send_as_bot( + token="test-token-placeholder", + text="bounded reply", + reply_to_message_id=11, + actual_bot_alias="openclaw_bot", + inbound_chat_id="verified-room", + ) + + assert result["_awooop_delivery_status"] == "sent" + canonical_send.assert_awaited_once() + method, payload = canonical_send.await_args.args + assert method == "sendMessage" + assert payload[gateway_module._ACTUAL_BOT_ALIAS_KEY] == "openclaw_bot" + assert payload[gateway_module._BOT_TOKEN_OVERRIDE_KEY] == ("test-token-placeholder") + assert payload[gateway_module._NON_MONITORING_INTERACTION_KEY] == { + "interaction_kind": "bot_discussion_reply", + "inbound_chat_id": "verified-room", + "inbound_message_id": 11, + } + + +@pytest.mark.asyncio +async def test_alternate_bot_binding_mismatch_blocks_before_http( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + monkeypatch.setattr( + gateway_module.settings, + "OPENCLAW_BOT_TOKEN", + "expected-placeholder", + ) + + result = await gateway._send_as_bot( + token="mismatched-placeholder", + text="must not send", + reply_to_message_id=11, + actual_bot_alias="openclaw_bot", + inbound_chat_id="test-room", + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awoooi_canonical_route_receipt"]["classifier"] == ( + "sender_bot_binding_mismatch" + ) + assert client.posts == [] + + +def test_readability_guard_uses_complete_ast_function_boundary() -> None: + guard = runpy.run_path( + str(_REPO_ROOT / "scripts" / "security" / "telegram-alert-readability-guard.py") + ) + long_body = "\n".join(f" value_{index} = {index}" for index in range(300)) + source = ( + 'DECOY = "async def _send_request"\n' + "class Gateway:\n" + " async def _send_request(self, payload):\n" + f"{long_body}\n" + " return normalize_telegram_send_message_payload(\n" + ' "sendMessage", payload\n' + " )\n\n" + " async def unrelated(self):\n" + " return False\n" + ) + + segment = guard["function_segment"]( + source, + "async def _send_request", + ) + + assert len(segment) > 2200 + assert "normalize_telegram_send_message_payload" in segment + assert "async def unrelated" not in segment + + @pytest.mark.asyncio async def test_operator_reply_lane_and_runtime_edit_binding_remain_available( monkeypatch: pytest.MonkeyPatch, diff --git a/apps/api/tests/test_telegram_delivery_truth_callers.py b/apps/api/tests/test_telegram_delivery_truth_callers.py index 604c6d904..8885d4f6c 100644 --- a/apps/api/tests/test_telegram_delivery_truth_callers.py +++ b/apps/api/tests/test_telegram_delivery_truth_callers.py @@ -13,6 +13,7 @@ from src.models.incident import Incident, Severity, Signal from src.repositories.alert_operation_log_repository import ALERT_EVENT_TYPES from src.services import decision_manager as decision_module from src.services import report_generation_service as report_module +from src.services import telegram_gateway as gateway_module from src.services.ai_rate_limiter import AIRateLimiter from src.services.approval_execution import ApprovalExecutionService from src.services.report_generation_service import ReportGenerationService @@ -27,11 +28,33 @@ NO_SEND_RECEIPT = { }, } +_SENT_CHAT_ID = -1001 +_SENT_DESTINATION_BINDING = gateway_module._telegram_destination_binding( + _SENT_CHAT_ID +) SENT_RECEIPT = { "ok": True, "_awooop_delivery_status": "sent", "_awooop_provider_send_performed": True, "result": {"message_id": 42, "chat": {"id": -1001}}, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "sender_bot_alias": "tsenyang_bot", + "provider_send_performed": True, + }, + gateway_module._DELIVERY_CONTEXT_KEY: { + "schema_version": "telegram_delivery_context_v1", + "method": "sendMessage", + "sender_bot_alias": "tsenyang_bot", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "payload_destination_binding": _SENT_DESTINATION_BINDING, + "provider_destination_binding": _SENT_DESTINATION_BINDING, + "destination_binding_verified": True, + }, } @@ -260,9 +283,7 @@ async def test_approval_blocked_receipt_writes_no_send_not_sent(monkeypatch) -> repair_attempted=True, ) - assert [row["event_type"] for row in operations] == [ - "NOTIFICATION_CLASSIFIED" - ] + assert [row["event_type"] for row in operations] == ["NOTIFICATION_CLASSIFIED"] assert operations[0]["event_type"] in ALERT_EVENT_TYPES assert operations[0]["action_detail"] == "telegram_execution_result_no_send" assert operations[0]["success"] is False diff --git a/apps/api/tests/test_telegram_delivery_truth_callers_v2.py b/apps/api/tests/test_telegram_delivery_truth_callers_v2.py index 841305d6b..18904e837 100644 --- a/apps/api/tests/test_telegram_delivery_truth_callers_v2.py +++ b/apps/api/tests/test_telegram_delivery_truth_callers_v2.py @@ -24,6 +24,7 @@ from src.services import converged_alert_recurrence_notifier as recurrence_modul from src.services import decision_manager as decision_module from src.services import failover_alerter as failover_module from src.services import k3s_monitor_service as k3s_module +from src.services import telegram_gateway as gateway_module from src.services import weekly_report_service as weekly_module NO_PROVIDER_SEND_RECEIPT = { @@ -36,11 +37,33 @@ NO_PROVIDER_SEND_RECEIPT = { }, } +_SENT_CHAT_ID = -1001 +_SENT_DESTINATION_BINDING = gateway_module._telegram_destination_binding( + _SENT_CHAT_ID +) SENT_RECEIPT = { "ok": True, "_awooop_delivery_status": "sent", "_awooop_provider_send_performed": True, "result": {"message_id": 42, "chat": {"id": -1001}}, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "sender_bot_alias": "tsenyang_bot", + "provider_send_performed": True, + }, + gateway_module._DELIVERY_CONTEXT_KEY: { + "schema_version": "telegram_delivery_context_v1", + "method": "sendMessage", + "sender_bot_alias": "tsenyang_bot", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "payload_destination_binding": _SENT_DESTINATION_BINDING, + "provider_destination_binding": _SENT_DESTINATION_BINDING, + "destination_binding_verified": True, + }, } @@ -170,7 +193,9 @@ async def test_watchdog_releases_dedup_and_does_not_log_sent_without_provider_ac logger = _Logger() monkeypatch.setattr(watchdog_module, "logger", logger) monkeypatch.setattr(watchdog_module, "get_redis", lambda: redis) - monkeypatch.setattr(watchdog_module, "_is_grace_active", AsyncMock(return_value=True)) + monkeypatch.setattr( + watchdog_module, "_is_grace_active", AsyncMock(return_value=True) + ) monkeypatch.setattr( watchdog_module, "_count_pending_no_tg_sent", diff --git a/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py index 82d5fd041..b92866146 100644 --- a/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py +++ b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py @@ -25,11 +25,33 @@ NO_SEND_RECEIPT = { }, } +_SENT_CHAT_ID = -1001 +_SENT_DESTINATION_BINDING = gateway_module._telegram_destination_binding( + _SENT_CHAT_ID +) SENT_RECEIPT = { "ok": True, "_awooop_delivery_status": "sent", "_awooop_provider_send_performed": True, - "result": {"message_id": 42}, + "result": {"message_id": 42, "chat": {"id": _SENT_CHAT_ID}}, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "sender_bot_alias": "tsenyang_bot", + "provider_send_performed": True, + }, + gateway_module._DELIVERY_CONTEXT_KEY: { + "schema_version": "telegram_delivery_context_v1", + "method": "sendMessage", + "sender_bot_alias": "tsenyang_bot", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "payload_destination_binding": _SENT_DESTINATION_BINDING, + "provider_destination_binding": _SENT_DESTINATION_BINDING, + "destination_binding_verified": True, + }, } @@ -73,9 +95,86 @@ class FakeRedis: return True -def test_delivery_helper_rejects_truthy_structured_no_send() -> None: - assert _telegram_send_delivery_succeeded(dict(NO_SEND_RECEIPT)) is False - assert _telegram_send_delivery_succeeded({"ok": True}) is True +@pytest.mark.parametrize( + ("receipt", "expected"), + [ + (SENT_RECEIPT, True), + (NO_SEND_RECEIPT, False), + ({"ok": True}, False), + ({**SENT_RECEIPT, "ok": 1}, False), + ({**SENT_RECEIPT, "_awooop_delivery_status": ""}, False), + ({**SENT_RECEIPT, "_awooop_provider_send_performed": None}, False), + ({**SENT_RECEIPT, "result": {}}, False), + ({**SENT_RECEIPT, "result": {"message_id": 0}}, False), + ({**SENT_RECEIPT, "result": {"message_id": True}}, False), + ({**SENT_RECEIPT, "result": {"message_id": "42"}}, False), + ({**SENT_RECEIPT, "_awoooi_canonical_route_receipt": {}}, False), + ({**SENT_RECEIPT, gateway_module._DELIVERY_CONTEXT_KEY: {}}, False), + ( + { + **SENT_RECEIPT, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "blocked_no_egress", + "provider_send_performed": True, + }, + }, + False, + ), + ( + { + **SENT_RECEIPT, + "_awoooi_canonical_route_receipt": { + **SENT_RECEIPT["_awoooi_canonical_route_receipt"], + "sender_bot_alias": "", + }, + }, + False, + ), + ( + { + **SENT_RECEIPT, + gateway_module._DELIVERY_CONTEXT_KEY: { + **SENT_RECEIPT[gateway_module._DELIVERY_CONTEXT_KEY], + "sender_bot_alias": "openclaw_bot", + }, + }, + False, + ), + ( + { + **SENT_RECEIPT, + "_awoooi_canonical_route_receipt": { + **SENT_RECEIPT["_awoooi_canonical_route_receipt"], + "destination_alias": "other_destination", + }, + }, + False, + ), + ( + { + **SENT_RECEIPT, + "_awoooi_canonical_route_receipt": { + **SENT_RECEIPT["_awoooi_canonical_route_receipt"], + "destination_binding": "", + }, + }, + False, + ), + ( + { + **SENT_RECEIPT, + "result": {"message_id": 42, "chat": {"id": -2002}}, + }, + False, + ), + ], +) +def test_delivery_helper_fails_closed_without_complete_canonical_receipt( + receipt: object, + expected: bool, +) -> None: + assert _telegram_send_delivery_succeeded(receipt) is expected @pytest.mark.asyncio @@ -244,7 +343,8 @@ async def test_html_callback_no_send_exhausts_fallback_and_records_failure( assert send_request.await_count == 3 record_failure.assert_awaited_once() statuses = [ - call.args[1].get(gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY, {}) + call.args[1] + .get(gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY, {}) .get("callback_reply", {}) .get("status") for call in send_request.await_args_list @@ -298,6 +398,7 @@ async def test_interactive_reply_no_send_logs_failure_not_sent( failed_event = "ai_advisory_group_reply_failed" sent_event = "ai_advisory_group_reply_sent" else: + class Repo: async def get_by_id(self, _incident_id: str): return None @@ -451,9 +552,7 @@ async def test_dev_test_push_returns_false_for_structured_no_send( monkeypatch.setattr(telegram.settings, "ENVIRONMENT", "dev") monkeypatch.setattr(telegram, "get_telegram_gateway", lambda: Gateway()) - response = await telegram.test_push( - telegram.TestPushRequest(approval_id="APR-1") - ) + response = await telegram.test_push(telegram.TestPushRequest(approval_id="APR-1")) assert response["ok"] is False assert response["message"] == "Test push no-send" diff --git a/apps/api/tests/test_telegram_message_templates.py b/apps/api/tests/test_telegram_message_templates.py index e57038bb2..51b0570a3 100644 --- a/apps/api/tests/test_telegram_message_templates.py +++ b/apps/api/tests/test_telegram_message_templates.py @@ -26,6 +26,35 @@ from src.services.telegram_gateway import ( normalize_telegram_send_message_payload, ) +_SENT_CHAT_ID = -1001 +_SENT_DESTINATION_BINDING = ( + telegram_gateway_module._telegram_destination_binding(_SENT_CHAT_ID) +) +_CANONICAL_SENT_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 77, "chat": {"id": _SENT_CHAT_ID}}, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "sender_bot_alias": "tsenyang_bot", + "provider_send_performed": True, + }, + telegram_gateway_module._DELIVERY_CONTEXT_KEY: { + "schema_version": "telegram_delivery_context_v1", + "method": "sendMessage", + "sender_bot_alias": "tsenyang_bot", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": _SENT_DESTINATION_BINDING, + "payload_destination_binding": _SENT_DESTINATION_BINDING, + "provider_destination_binding": _SENT_DESTINATION_BINDING, + "destination_binding_verified": True, + }, +} + def test_auto_repair_status_line_distinguishes_ai_retry_queued() -> None: """自動化失敗 reply 必須明確標示 AI 續跑，且不把 raw error 當純文字噴出。""" @@ -289,7 +318,7 @@ async def test_send_alert_notification_normalizes_host_resource_raw_dump(monkeyp async def fake_send_request(method, payload): sent_requests.append((method, payload)) - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat")) monkeypatch.setattr(gateway, "_send_request", fake_send_request) @@ -329,7 +358,7 @@ async def test_send_alert_notification_normalizes_aiops_signal_alert(monkeypatch async def fake_send_request(method, payload): sent_requests.append((method, payload)) - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat")) monkeypatch.setattr(gateway, "_send_request", fake_send_request) @@ -384,7 +413,7 @@ async def test_send_alert_notification_forces_html_card_for_markdown_host_alert( async def fake_send_request(method, payload): sent_requests.append((method, payload)) - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat")) monkeypatch.setattr(gateway, "_send_request", fake_send_request) @@ -417,7 +446,7 @@ async def test_send_text_normalizes_host_resource_alert(monkeypatch) -> None: async def fake_send_request(method, payload): sent_requests.append((method, payload)) - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat")) monkeypatch.setattr(gateway, "_send_request", fake_send_request) @@ -482,7 +511,10 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch) return None def json(self) -> dict: - return {"ok": True, "result": {"message_id": 991}} + return { + "ok": True, + "result": {"message_id": 991, "chat": {"id": "chat"}}, + } class FakeHttpClient: async def post(self, url, json): # type: ignore[no-untyped-def] @@ -541,7 +573,10 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch) await asyncio.sleep(0) assert result["ok"] is True - assert result["result"] == {"message_id": 991} + assert result["result"] == { + "message_id": 991, + "chat": {"id": "chat"}, + } assert result["_awoooi_canonical_route_receipt"]["decision"] == "allowed" assert mirrored assert mirrored[0]["source"] == "send_request" @@ -1789,7 +1824,10 @@ async def test_send_request_strips_awooop_callback_metadata_before_telegram_api( return None def json(self): - return {"ok": True, "result": {"message_id": 456}} + return { + "ok": True, + "result": {"message_id": 456, "chat": {"id": "chat"}}, + } class FakeClient: async def post(self, url, json): @@ -1878,7 +1916,7 @@ async def test_send_html_line_message_falls_back_to_plain_text_on_parse_error(mo sent_requests.append((method, payload)) if payload.get("parse_mode") == "HTML": raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400") - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(gateway, "_send_request", fake_send_request) @@ -1910,7 +1948,7 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) sent_requests.append((method, payload)) if payload.get("parse_mode") == "HTML": raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400") - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(gateway, "_send_request", fake_send_request) @@ -2002,7 +2040,13 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) acknowledged = ( telegram_gateway_module._acknowledge_callback_reply_source_envelope( fallback_source_extra, - delivery_result={"ok": True}, + delivery_result={ + **_CANONICAL_SENT_RECEIPT, + "result": { + "message_id": 88, + "chat": {"id": _SENT_CHAT_ID}, + }, + }, provider_message_id="88", ) ) @@ -2042,7 +2086,7 @@ async def test_send_html_line_message_uses_rescue_when_markup_fallback_fails(mon sent_requests.append((method, payload)) if len(sent_requests) < 3: raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400") - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(gateway, "_send_request", fake_send_request) @@ -2070,7 +2114,7 @@ async def test_send_html_line_message_attaches_awooop_markup_to_first_chunk(monk async def fake_send_request(method, payload): sent_requests.append((method, payload)) - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(gateway, "_send_request", fake_send_request) @@ -2160,7 +2204,7 @@ async def test_send_notification_long_html_uses_plain_text_without_cutting_tags( async def fake_send_request(method, payload): sent_requests.append((method, payload)) - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(gateway, "_send_request", fake_send_request) long_text = "📊 事件歷史統計\n告警鍵: " + ("node<188>&" * 90) + "" @@ -2446,7 +2490,7 @@ async def test_append_incident_update_deduplicates_same_status(monkeypatch): async def fake_send_request(method, payload): sent_requests.append((method, payload)) - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: fake_redis) monkeypatch.setattr(gateway, "_send_request", fake_send_request) @@ -2491,7 +2535,7 @@ async def test_append_incident_update_suppresses_duplicate_failure_across_incide async def fake_send_request(method, payload): sent_requests.append((method, payload)) - return {"ok": True} + return dict(_CANONICAL_SENT_RECEIPT) monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: fake_redis) monkeypatch.setattr(gateway, "_send_request", fake_send_request) diff --git a/apps/api/tests/test_telegram_notification_egress_scanners.py b/apps/api/tests/test_telegram_notification_egress_scanners.py new file mode 100644 index 000000000..3959d99d5 --- /dev/null +++ b/apps/api/tests/test_telegram_notification_egress_scanners.py @@ -0,0 +1,495 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +import pytest + +ROOT = Path(__file__).resolve().parents[3] + + +def _load_scanner(filename: str, module_name: str) -> ModuleType: + path = ROOT / "scripts/security" / filename + spec = importlib.util.spec_from_file_location(module_name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +INVENTORY = _load_scanner( + "telegram-notification-egress-inventory.py", + "telegram_notification_egress_inventory_test", +) +NO_NEW_BYPASS_GUARD = _load_scanner( + "telegram-notification-egress-no-new-bypass-guard.py", + "telegram_notification_egress_no_new_bypass_guard_test", +) +SCANNERS = (INVENTORY, NO_NEW_BYPASS_GUARD) + + +def _write_zero_baseline(root: Path) -> None: + snapshot = ( + root / "docs/security/telegram-notification-egress-inventory.snapshot.json" + ) + snapshot.parent.mkdir(parents=True, exist_ok=True) + snapshot.write_text( + json.dumps( + { + "summary": { + "direct_bot_api_call_count": 0, + "direct_bot_api_file_count": 0, + }, + "direct_bot_api_calls": [], + } + ), + encoding="utf-8", + ) + + +def _write_minimal_gateway(root: Path) -> None: + gateway = root / "apps/api/src/services/telegram_gateway.py" + gateway.parent.mkdir(parents=True, exist_ok=True) + gateway.write_text( + "normalize_telegram_send_message_payload = object()\n", + encoding="utf-8", + ) + + +@pytest.mark.parametrize("scanner", SCANNERS) +def test_scanner_detects_multiline_split_bot_api_url(scanner: ModuleType) -> None: + source = """ +import requests + +def send_telegram_alert(token: str) -> object: + base = "https://api.telegram.org/" + endpoint = ( + base + + f"bot{token}/" + + "sendMessage" + ) + return requests.post(endpoint, json={"text": "bounded"}) +""" + + findings = scanner.scan_direct_bot_api_surfaces("scripts/ops/split.py", source) + + assert len(findings) == 1 + assert findings[0]["method"] == "sendMessage" + assert findings[0]["detection_kind"] == "direct_bot_api_endpoint" + + +@pytest.mark.parametrize("scanner", SCANNERS) +@pytest.mark.parametrize("transport", ["Invoke-WebRequest", "Invoke-RestMethod"]) +def test_scanner_detects_powershell_direct_transport( + scanner: ModuleType, + transport: str, +) -> None: + source = f""" +function Send-AgentTelegramLegacy {{ + $base = "https://api.telegram.org/" + $uri = $base + "bot" + $Token + "/sendPhoto" + {transport} -Method Post -Uri $uri +}} +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "agent99-control-plane.ps1", + source, + ) + + assert len(findings) == 1 + assert findings[0]["method"] == "sendPhoto" + assert findings[0]["detection_kind"] == "direct_bot_api_endpoint" + + +@pytest.mark.parametrize("scanner", SCANNERS) +@pytest.mark.parametrize( + "transport", + [ + "requests.post(target, json=payload)", + "urllib.request.urlopen(target, data=payload)", + "client.PostAsync(target, content)", + ], +) +def test_scanner_detects_custom_sender_without_literal_endpoint( + scanner: ModuleType, + transport: str, +) -> None: + source = f""" +function Send-AgentTelegramLegacy {{ + $target = $ConfiguredProviderUrl + "/sendMessage" + {transport} +}} +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "agent99-control-plane.ps1", + source, + ) + + assert len(findings) == 1 + assert findings[0]["method"] == "sendMessage" + assert findings[0]["detection_kind"] == "custom_direct_sender" + + +@pytest.mark.parametrize("scanner", SCANNERS) +@pytest.mark.parametrize( + "transport", + [ + "httpx.Client().post(endpoint, json=payload)", + "requests.Session().post(endpoint, json=payload)", + "urllib.request.build_opener().open(endpoint, data=payload)", + "opener.open(endpoint, data=payload)", + ], +) +def test_scanner_detects_factory_clients_and_urllib_openers_with_dynamic_endpoint( + scanner: ModuleType, + transport: str, +) -> None: + source = f""" +def dispatch(endpoint: str, bot_token: str, chat_id: str, payload: bytes) -> object: + return {transport} +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "scripts/ops/provider_dispatch.py", + source, + ) + + assert len(findings) == 1 + assert findings[0]["method"] == "dynamic" + assert findings[0]["detection_kind"] == "custom_direct_sender" + assert findings[0]["function"] == "dispatch" + + +@pytest.mark.parametrize("scanner", SCANNERS) +@pytest.mark.parametrize( + ("imports", "transport"), + [ + ("import requests as rq", "rq.post(endpoint, json=payload)"), + ("import httpx as hx", "hx.post(endpoint, json=payload)"), + ( + "import httpx as hx", + "hx.Client().post(endpoint, json=payload)", + ), + ( + "from httpx import Client as HC", + "HC().post(endpoint, json=payload)", + ), + ( + "from requests import Session as S", + "S().post(endpoint, json=payload)", + ), + ( + "from urllib.request import build_opener as make_opener", + "make_opener().open(endpoint, data=payload)", + ), + ], +) +def test_scanner_resolves_python_import_alias_transport_calls( + scanner: ModuleType, + imports: str, + transport: str, +) -> None: + source = f""" +{imports} + +def dispatch(endpoint: str, bot_token: str, chat_id: str, payload: bytes) -> object: + return {transport} +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "scripts/ops/aliased_provider_dispatch.py", + source, + ) + + assert len(findings) == 1 + assert findings[0]["method"] == "dynamic" + assert findings[0]["detection_kind"] == "custom_direct_sender" + assert findings[0]["function"] == "dispatch" + + +@pytest.mark.parametrize("scanner", SCANNERS) +@pytest.mark.parametrize( + ("imports", "transport"), + [ + ("import requests as rq", "rq.post(endpoint, json=payload)"), + ( + "import httpx as hx", + "hx.Client().post(endpoint, json=payload)", + ), + ( + "from httpx import Client as HC", + "HC().post(endpoint, json=payload)", + ), + ( + "from requests import Session as S", + "S().post(endpoint, json=payload)", + ), + ( + "from urllib.request import build_opener as make_opener", + "make_opener().open(endpoint, data=payload)", + ), + ], +) +def test_scanner_resolves_module_scope_import_alias_transport_calls( + scanner: ModuleType, + imports: str, + transport: str, +) -> None: + source = f""" +{imports} + +bot_token = object() +chat_id = object() +endpoint = object() +payload = b"" +result = {transport} +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "scripts/ops/module_alias_provider.py", + source, + ) + + assert len(findings) == 1 + assert findings[0]["method"] == "dynamic" + assert findings[0]["detection_kind"] == "custom_direct_sender" + assert findings[0]["function"] == "" + assert findings[0]["function_qualified"] == "" + + +@pytest.mark.parametrize("scanner", SCANNERS) +def test_scanner_resolves_imported_factory_assigned_to_local_client( + scanner: ModuleType, +) -> None: + source = """ +from httpx import Client as HC + +def dispatch(endpoint: str, bot_token: str, chat_id: str, payload: bytes) -> object: + sender = HC() + return sender.post(endpoint, content=payload) +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "scripts/ops/assigned_alias_provider.py", + source, + ) + + assert len(findings) == 1 + assert findings[0]["method"] == "dynamic" + assert findings[0]["function"] == "dispatch" + + +@pytest.mark.parametrize("scanner", SCANNERS) +@pytest.mark.parametrize("upload_method", ["UploadString", "UploadData", "UploadFile"]) +def test_scanner_detects_powershell_webclient_uploads_with_dynamic_endpoint( + scanner: ModuleType, + upload_method: str, +) -> None: + source = f""" +function Publish-LegacyPayload {{ + param($BotToken, $ChatId, $Uri) + $webClient = New-Object System.Net.WebClient + $webClient.{upload_method}($Uri, $Payload) +}} +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "scripts/ops/legacy-provider.ps1", + source, + ) + + assert len(findings) == 1 + assert findings[0]["method"] == "dynamic" + assert findings[0]["detection_kind"] == "custom_direct_sender" + assert findings[0]["function"] == "Publish-LegacyPayload" + + +@pytest.mark.parametrize("scanner", SCANNERS) +def test_scanner_excludes_only_canonical_gateway_final_exit( + scanner: ModuleType, +) -> None: + source = """ +class TelegramGateway: + async def _send_request(self, token: str, payload: dict) -> object: + url = f"https://api.telegram.org/bot{token}/sendMessage" + return await self._http_client.post(url, json=payload) +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "apps/api/src/services/telegram_gateway.py", + source, + ) + + assert findings == [] + + +@pytest.mark.parametrize("scanner", SCANNERS) +def test_scanner_does_not_allowlist_same_function_name_outside_gateway( + scanner: ModuleType, +) -> None: + source = """ +import requests as rq + +async def _send_request( + endpoint: str, + bot_token: str, + chat_id: str, + payload: dict, +) -> object: + return rq.post(endpoint, json=payload) +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "scripts/ops/telegram_gateway.py", + source, + ) + + assert len(findings) == 1 + assert findings[0]["function"] == "_send_request" + assert findings[0]["detection_kind"] == "custom_direct_sender" + + +@pytest.mark.parametrize("scanner", SCANNERS) +def test_scanner_allowlists_only_class_qualified_gateway_final_exit( + scanner: ModuleType, +) -> None: + source = """ +import requests as rq + +async def _send_request( + endpoint: str, + bot_token: str, + chat_id: str, + payload: dict, +) -> object: + return rq.post(endpoint, json=payload) + +class RogueGateway: + async def _send_request( + self, + endpoint: str, + bot_token: str, + chat_id: str, + payload: dict, + ) -> object: + return rq.post(endpoint, json=payload) + +class TelegramGateway: + async def _send_request( + self, + endpoint: str, + bot_token: str, + chat_id: str, + payload: dict, + ) -> object: + return rq.post(endpoint, json=payload) +""" + + findings = scanner.scan_direct_bot_api_surfaces( + "apps/api/src/services/telegram_gateway.py", + source, + ) + + assert {item["function_qualified"] for item in findings} == { + "_send_request", + "RogueGateway._send_request", + } + + +def test_no_new_bypass_guard_fails_closed_on_split_endpoint(tmp_path: Path) -> None: + _write_zero_baseline(tmp_path) + _write_minimal_gateway(tmp_path) + sender = tmp_path / "scripts/ops/split_sender.py" + sender.parent.mkdir(parents=True, exist_ok=True) + sender.write_text( + """ +import requests + +def send_telegram(token: str) -> object: + base = "https://api.telegram.org/" + endpoint = base + f"bot{token}/" + "sendDocument" + return requests.post(endpoint, files={}) +""", + encoding="utf-8", + ) + + report = NO_NEW_BYPASS_GUARD.build_report(tmp_path) + + assert report["summary"]["current_direct_bot_api_call_count"] == 1 + assert report["summary"]["new_bypass_count"] == 1 + with pytest.raises(SystemExit, match="direct/custom bypass"): + NO_NEW_BYPASS_GUARD.validate(tmp_path) + + +def test_local_github_workflow_is_frozen_legacy_not_active_bypass( + tmp_path: Path, +) -> None: + _write_zero_baseline(tmp_path) + _write_minimal_gateway(tmp_path) + workflow = tmp_path / ".github/workflows/legacy-alert.yml" + workflow.parent.mkdir(parents=True, exist_ok=True) + workflow.write_text( + """ +steps: + - name: frozen legacy telegram sender + run: | + endpoint="https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" + curl -X POST "$endpoint" -d "chat_id=${CHAT_ID}" +""", + encoding="utf-8", + ) + + inventory_report = INVENTORY.build_report(tmp_path) + inventory_item = inventory_report["direct_bot_api_calls"][0] + assert inventory_report["summary"]["direct_bot_api_call_count"] == 1 + assert inventory_report["summary"]["active_direct_bot_api_call_count"] == 0 + assert ( + inventory_report["summary"]["github_frozen_legacy_direct_bot_api_call_count"] + == 1 + ) + assert inventory_item["source_truth_classification"] == ( + "frozen_legacy_source_truth" + ) + assert inventory_item["runtime_execution_authorized"] is False + assert inventory_item["workflow_execution_authorized"] is False + assert ( + inventory_report["execution_boundaries"]["github_workflow_execution_authorized"] + is False + ) + + guard_report = NO_NEW_BYPASS_GUARD.build_report(tmp_path) + assert guard_report["status"] == "pass_no_direct_or_custom_bypass" + assert guard_report["summary"]["detected_direct_bot_api_call_count"] == 1 + assert guard_report["summary"]["current_direct_bot_api_call_count"] == 0 + assert guard_report["summary"]["new_bypass_count"] == 0 + assert ( + guard_report["summary"]["github_frozen_legacy_direct_bot_api_call_count"] == 1 + ) + frozen_item = guard_report["frozen_legacy_direct_bot_api_calls"][0] + assert frozen_item["source_truth_classification"] == ("frozen_legacy_source_truth") + assert frozen_item["runtime_execution_authorized"] is False + assert frozen_item["workflow_execution_authorized"] is False + assert ( + guard_report["execution_boundaries"]["github_workflow_execution_authorized"] + is False + ) + NO_NEW_BYPASS_GUARD.validate(tmp_path) + + +def test_agent99_has_no_direct_or_custom_telegram_sender_source() -> None: + source = (ROOT / "agent99-control-plane.ps1").read_text(encoding="utf-8") + + for scanner in SCANNERS: + assert ( + scanner.scan_direct_bot_api_surfaces("agent99-control-plane.ps1", source) + == [] + ) + assert "function Send-AgentTelegramRelay" not in source + assert "function Send-AgentTelegramPhotoDirect" not in source + assert "canonical_telegram_gateway_transport_required" in source + assert "providerSendPerformed = $false" in source + assert 'routeStatus = "blocked_no_egress"' in source diff --git a/scripts/security/telegram-alert-readability-guard.py b/scripts/security/telegram-alert-readability-guard.py index 34b4346e2..9d4041df2 100644 --- a/scripts/security/telegram-alert-readability-guard.py +++ b/scripts/security/telegram-alert-readability-guard.py @@ -8,6 +8,7 @@ Bot API、不讀 secret、不連線主機，也不啟動任何 runtime gate。 from __future__ import annotations import argparse +import ast import json import subprocess from datetime import datetime, timedelta, timezone @@ -164,11 +165,44 @@ def require_contains(label: str, text: str, marker: str) -> None: raise SystemExit(f"BLOCKED {label}: missing {marker!r}") -def function_segment(text: str, marker: str, *, limit: int = 2200) -> str: - start = text.find(marker) - if start == -1: - raise SystemExit(f"BLOCKED source function marker missing: {marker!r}") - return text[start : start + limit] +def function_segment(text: str, marker: str) -> str: + """Return the complete named function using Python AST source boundaries.""" + marker_prefix, separator, function_name = marker.partition("def ") + function_name = function_name.strip() + if not separator or not function_name.isidentifier(): + raise SystemExit(f"BLOCKED invalid source function marker: {marker!r}") + + expected_type: type[ast.FunctionDef] | type[ast.AsyncFunctionDef] + expected_type = ( + ast.AsyncFunctionDef + if marker_prefix.strip() == "async" + else ast.FunctionDef + ) + try: + tree = ast.parse(text) + except SyntaxError as exc: + raise SystemExit( + f"BLOCKED telegram gateway source is not valid Python: {exc.msg}" + ) from exc + + matches = [ + node + for node in ast.walk(tree) + if isinstance(node, expected_type) and node.name == function_name + ] + if len(matches) != 1: + raise SystemExit( + "BLOCKED source function marker must resolve exactly once: " + f"{marker!r}, matches={len(matches)}" + ) + + node = matches[0] + if node.end_lineno is None: + raise SystemExit( + f"BLOCKED source function boundary unavailable: {marker!r}" + ) + lines = text.splitlines(keepends=True) + return "".join(lines[node.lineno - 1 : node.end_lineno]) def git_commit(root: Path) -> str: diff --git a/scripts/security/telegram-notification-egress-inventory.py b/scripts/security/telegram-notification-egress-inventory.py index ec392c273..14e2a157c 100644 --- a/scripts/security/telegram-notification-egress-inventory.py +++ b/scripts/security/telegram-notification-egress-inventory.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 """Build a repo-only Telegram notification egress inventory. -This scanner identifies Telegram Bot API sendMessage paths that can bypass -TelegramGateway's final-exit formatter. It does not read secrets, call -Telegram, modify workflows, or send notifications. +This scanner identifies guarded Telegram Bot API paths that can bypass +TelegramGateway's final-exit formatter. Local ``.github/workflows`` files are +audited only as frozen legacy source truth. It does not read secrets, call +Telegram, modify or execute workflows, or send notifications. """ from __future__ import annotations import argparse +import ast import hashlib import json import re @@ -24,6 +26,7 @@ TAIPEI = timezone(timedelta(hours=8)) SCAN_ROOTS = ( Path("agent99-control-plane.ps1"), Path(".gitea/workflows"), + Path(".github/workflows"), Path("scripts/ops"), Path("scripts/ci"), Path("scripts/reboot-recovery"), @@ -42,17 +45,81 @@ GUARDED_BOT_METHODS = ( "sendAudio", "sendVoice", ) -DIRECT_BOT_API_RE = re.compile( +BOT_TOKEN_URL_RE = re.compile( r"api\.telegram\.org/bot.*?/(?P" + "|".join(re.escape(method) for method in GUARDED_BOT_METHODS) + r")\b", re.IGNORECASE, ) +GUARDED_BOT_METHOD_RE = re.compile( + r"\b(?P" + + "|".join(re.escape(method) for method in GUARDED_BOT_METHODS) + + r")\b", + re.IGNORECASE, +) +COMPACT_BOT_ENDPOINT_RE = re.compile( + r"api\.telegram\.org.{0,1024}?bot.{0,512}?/(?P" + + "|".join(re.escape(method) for method in GUARDED_BOT_METHODS) + + r")", + re.IGNORECASE, +) +DIRECT_HTTP_TRANSPORT_RE = re.compile( + r"(?:" + r"requests\s*\.\s*Session\s*\([^)]*\)\s*\.\s*(?:post|request|send)\s*\(|" + r"requests\s*\.\s*(?:post|request)\s*\(|" + r"httpx\s*\.\s*(?:AsyncClient|Client)\s*\([^)]*\)\s*\.\s*(?:post|request|send)\s*\(|" + r"httpx\s*\.\s*(?:post|request)\s*\(|" + r"(?:urllib\s*\.\s*request\s*\.\s*)?build_opener\s*\([^)]*\)\s*\.\s*open\s*\(|" + r"urllib\s*\.\s*request\s*\.\s*(?:urlopen|Request)\s*\(|" + r"\burlopen\s*\(|" + r"\b(?:_?opener)\s*\.\s*open\s*\(|" + r"\bInvoke-(?:WebRequest|RestMethod)\b|" + r"\.\s*PostAsync\s*\(|" + r"\.\s*Upload(?:String|Data|File)\s*\(|" + r"\b(?:fetch|curl|wget)\b|" + r"\b(?:_http_client|client|session)\s*\.\s*(?:post|request|send)\s*\(" + r")", + re.IGNORECASE, +) +TELEGRAM_CONTEXT_RE = re.compile( + r"(?:telegram|bot[_-]?token|chat[_-]?id)", re.IGNORECASE +) +BOT_TOKEN_CONTEXT_RE = re.compile( + r"(?:\bbot[_-]?token\b|\btelegram[_-]?(?:bot[_-]?)?token\b|\$Token\b)", + re.IGNORECASE, +) +CHAT_CONTEXT_RE = re.compile( + r"(?:\bchat[_-]?id\b|\$ChatId\b)", + re.IGNORECASE, +) +POWERSHELL_FUNCTION_RE = re.compile( + r"^function\s+(?P[A-Za-z0-9_-]+)\s*\{", re.IGNORECASE | re.MULTILINE +) +CANONICAL_FINAL_EXITS = { + ( + "apps/api/src/services/telegram_gateway.py", + "TelegramGateway._send_request", + ), +} +_AST_IMPORT_NODE_TYPES = (ast.Import, ast.ImportFrom) +_AST_FUNCTION_NODE_TYPES = (ast.FunctionDef, ast.AsyncFunctionDef) +_AST_SEQUENCE_TARGET_NODE_TYPES = (ast.Tuple, ast.List) +_AST_NON_MODULE_SCOPE_NODE_TYPES = ( + ast.ClassDef, + ast.FunctionDef, + ast.AsyncFunctionDef, + ast.Lambda, +) GATEWAY_CALLSITE_RE = re.compile( r"(?:send_alert_notification\(|\b(?:tg|gw|gateway|telegram)\.send_text\(|_send_request\(\s*[\"']sendMessage[\"'])" ) SECRET_INTERPOLATION_RE = re.compile(r"\$\{\{\s*secrets\.[^}]+\}\}") -BOT_TOKEN_URL_RE = DIRECT_BOT_API_RE +BOT_TOKEN_FRAGMENT_RE = re.compile( + r"bot[^/\s\"']+/(?P" + + "|".join(re.escape(method) for method in GUARDED_BOT_METHODS) + + r")\b", + re.IGNORECASE, +) REQUIRED_OWNER_FIELDS = [ "egress_surface_id", @@ -167,10 +234,429 @@ def sanitize_excerpt(line: str) -> str: lambda match: f"api.telegram.org/bot/{match.group('method')}", excerpt, ) + excerpt = BOT_TOKEN_FRAGMENT_RE.sub( + lambda match: f"bot/{match.group('method')}", + excerpt, + ) return excerpt[:180] +def _compact_source(source: str) -> str: + without_string_prefixes = re.sub( + r"(?i)\b(?:fr|rf|f|r|u|b)(?=[\"'])", + "", + source, + ) + return re.sub(r"[\s\"'`+()]+", "", without_string_prefixes) + + +_PYTHON_HTTP_TRANSPORT_CALL_RE = re.compile( + r"^(?:" + r"requests\.(?:post|request)|" + r"requests\.Session\(\)\.(?:post|request|send)|" + r"httpx\.(?:post|request)|" + r"httpx\.(?:AsyncClient|Client)\(\)\.(?:post|request|send)|" + r"urllib\.request\.(?:urlopen|Request)|" + r"urllib\.request\.build_opener\(\)\.open" + r")$" +) + + +def _record_python_import_alias( + node: ast.AST, + aliases: dict[str, str], +) -> None: + if isinstance(node, ast.Import): + for imported in node.names: + bound_name = imported.asname or imported.name.split(".", 1)[0] + aliases[bound_name] = ( + imported.name if imported.asname else bound_name + ) + return + + module = str(node.module or "").strip(".") + if not module: + return + for imported in node.names: + if imported.name == "*": + continue + aliases[imported.asname or imported.name] = ( + f"{module}.{imported.name}" + ) + + +def _resolve_python_expression( + node: ast.AST, + aliases: dict[str, str], +) -> str | None: + if isinstance(node, ast.Name): + return aliases.get(node.id, node.id) + if isinstance(node, ast.Attribute): + owner = _resolve_python_expression(node.value, aliases) + return f"{owner}.{node.attr}" if owner else None + if isinstance(node, ast.Call): + callable_name = _resolve_python_expression(node.func, aliases) + return f"{callable_name}()" if callable_name else None + return None + + +def _assignment_target_names(node: ast.AST) -> list[str]: + if isinstance(node, ast.Name): + return [node.id] + if isinstance(node, _AST_SEQUENCE_TARGET_NODE_TYPES): + return [ + name + for child in node.elts + for name in _assignment_target_names(child) + ] + return [] + + +def _python_parent_map(tree: ast.AST) -> dict[ast.AST, ast.AST]: + return { + child: parent + for parent in ast.walk(tree) + for child in ast.iter_child_nodes(parent) + } + + +def _python_function_qualified_name( + function: ast.FunctionDef | ast.AsyncFunctionDef, + parents: dict[ast.AST, ast.AST], +) -> str: + parts = [function.name] + parent = parents.get(function) + while parent is not None: + if isinstance(parent, (ast.ClassDef, *_AST_FUNCTION_NODE_TYPES)): + parts.append(parent.name) + parent = parents.get(parent) + return ".".join(reversed(parts)) + + +def _nearest_python_function( + node: ast.AST, + parents: dict[ast.AST, ast.AST], +) -> ast.FunctionDef | ast.AsyncFunctionDef | None: + parent = parents.get(node) + while parent is not None: + if isinstance(parent, _AST_FUNCTION_NODE_TYPES): + return parent + parent = parents.get(parent) + return None + + +def _is_python_module_scope( + node: ast.AST, + parents: dict[ast.AST, ast.AST], +) -> bool: + parent = parents.get(node) + while parent is not None: + if isinstance(parent, _AST_NON_MODULE_SCOPE_NODE_TYPES): + return False + parent = parents.get(parent) + return True + + +def _resolve_python_transport_calls( + nodes: list[ast.AST], + aliases: dict[str, str], +) -> list[tuple[int, str]]: + resolved_calls: list[tuple[int, str]] = [] + for node in sorted( + nodes, + key=lambda item: ( + int(getattr(item, "lineno", 0)), + int(getattr(item, "col_offset", 0)), + 0 if isinstance(item, _AST_IMPORT_NODE_TYPES) else 1, + ), + ): + if isinstance(node, _AST_IMPORT_NODE_TYPES): + _record_python_import_alias(node, aliases) + continue + if isinstance(node, ast.Assign): + resolved_value = _resolve_python_expression(node.value, aliases) + for target in node.targets: + for name in _assignment_target_names(target): + if resolved_value: + aliases[name] = resolved_value + else: + aliases.pop(name, None) + continue + if isinstance(node, ast.AnnAssign): + resolved_value = ( + _resolve_python_expression(node.value, aliases) + if node.value is not None + else None + ) + for name in _assignment_target_names(node.target): + if resolved_value: + aliases[name] = resolved_value + else: + aliases.pop(name, None) + continue + if not isinstance(node, ast.Call): + continue + qualified_call = _resolve_python_expression(node.func, aliases) + if qualified_call and _PYTHON_HTTP_TRANSPORT_CALL_RE.fullmatch( + qualified_call + ): + resolved_calls.append((node.lineno, qualified_call)) + return sorted(set(resolved_calls)) + + +def _python_transport_calls_by_scope( + text: str, +) -> tuple[ + dict[tuple[str, int, int], list[tuple[int, str]]], + list[tuple[int, str]], +]: + """Resolve HTTP aliases in function and true module-level AST scopes.""" + try: + tree = ast.parse(text) + except SyntaxError: + return {}, [] + + parents = _python_parent_map(tree) + module_aliases: dict[str, str] = {} + module_nodes = [ + node + for node in ast.walk(tree) + if _is_python_module_scope(node, parents) + ] + module_calls = _resolve_python_transport_calls( + module_nodes, + module_aliases, + ) + + transports: dict[tuple[str, int, int], list[tuple[int, str]]] = {} + function_nodes = [ + node + for node in ast.walk(tree) + if isinstance(node, _AST_FUNCTION_NODE_TYPES) + and node.end_lineno is not None + ] + for function in function_nodes: + aliases = dict(module_aliases) + arguments = [ + *function.args.posonlyargs, + *function.args.args, + *function.args.kwonlyargs, + ] + if function.args.vararg is not None: + arguments.append(function.args.vararg) + if function.args.kwarg is not None: + arguments.append(function.args.kwarg) + for argument in arguments: + aliases.pop(argument.arg, None) + + scoped_nodes = [ + node + for node in ast.walk(function) + if node is function + or _nearest_python_function(node, parents) is function + ] + resolved_calls = _resolve_python_transport_calls(scoped_nodes, aliases) + + if resolved_calls: + qualified_name = _python_function_qualified_name(function, parents) + transports[ + ( + qualified_name, + function.lineno, + int(function.end_lineno), + ) + ] = resolved_calls + return transports, module_calls + + +def _transport_window_units( + text: str, + *, + excluded_line_ranges: list[tuple[int, int]], +) -> list[tuple[str, str, int, int, str]]: + lines = text.splitlines() + units: list[tuple[str, str, int, int, str]] = [] + for match in DIRECT_HTTP_TRANSPORT_RE.finditer(text): + line_number = text.count("\n", 0, match.start()) + 1 + if any(start <= line_number <= end for start, end in excluded_line_ranges): + continue + start_line = max(1, line_number - 20) + end_line = min(len(lines), line_number + 20) + units.append( + ( + "", + "", + start_line, + end_line, + "\n".join(lines[start_line - 1 : end_line]), + ) + ) + return units + + +def _source_units( + relative_path: str, + text: str, +) -> list[tuple[str, str, int, int, str]]: + lines = text.splitlines() + units: list[tuple[str, str, int, int, str]] = [] + ranges: list[tuple[int, int]] = [] + if relative_path.endswith(".py"): + try: + tree = ast.parse(text) + except SyntaxError: + tree = None + if tree is not None: + parents = _python_parent_map(tree) + nodes = [ + node + for node in ast.walk(tree) + if isinstance(node, _AST_FUNCTION_NODE_TYPES) + and getattr(node, "end_lineno", None) + ] + for node in sorted( + nodes, key=lambda item: (item.lineno, item.end_lineno or item.lineno) + ): + end_line = int(node.end_lineno or node.lineno) + ranges.append((node.lineno, end_line)) + units.append( + ( + node.name, + _python_function_qualified_name(node, parents), + node.lineno, + end_line, + "\n".join(lines[node.lineno - 1 : end_line]), + ) + ) + elif relative_path.endswith(".ps1"): + matches = list(POWERSHELL_FUNCTION_RE.finditer(text)) + for index, match in enumerate(matches): + start_line = text.count("\n", 0, match.start()) + 1 + end_offset = ( + matches[index + 1].start() if index + 1 < len(matches) else len(text) + ) + end_line = text.count("\n", 0, end_offset) + 1 + ranges.append((start_line, end_line)) + units.append( + ( + match.group("name"), + match.group("name"), + start_line, + end_line, + text[match.start() : end_offset], + ) + ) + return units + _transport_window_units(text, excluded_line_ranges=ranges) + + +def scan_direct_bot_api_surfaces( + relative_path: str, + text: str, +) -> list[dict[str, Any]]: + """Find direct Telegram transports, including split URLs and custom senders.""" + + source_lines = text.splitlines() + findings: list[dict[str, Any]] = [] + seen: set[tuple[int, str, str]] = set() + python_transports, module_transports = ( + _python_transport_calls_by_scope(text) + if relative_path.endswith(".py") + else ({}, []) + ) + source_units = _source_units(relative_path, text) + for line_number, _qualified_call in module_transports: + start_line = max(1, line_number - 20) + end_line = min(len(source_lines), line_number + 20) + source_units.append( + ( + "", + "", + start_line, + end_line, + "\n".join(source_lines[start_line - 1 : end_line]), + ) + ) + + for ( + function_name, + function_qualified, + start_line, + _end_line, + source, + ) in source_units: + if (relative_path, function_qualified) in CANONICAL_FINAL_EXITS: + continue + resolved_python_transports = ( + [ + item + for item in module_transports + if start_line <= item[0] <= _end_line + ] + if function_qualified == "" + else python_transports.get( + (function_qualified, start_line, _end_line), + [], + ) + ) + transport_match = DIRECT_HTTP_TRANSPORT_RE.search(source) + if transport_match is None and not resolved_python_transports: + continue + method_match = GUARDED_BOT_METHOD_RE.search(source) + endpoint_match = COMPACT_BOT_ENDPOINT_RE.search(_compact_source(source)) + telegram_named = bool( + re.search(r"(?:telegram|bot)", function_name, re.IGNORECASE) + ) + telegram_context = bool(TELEGRAM_CONTEXT_RE.search(source)) + token_and_chat_context = bool( + BOT_TOKEN_CONTEXT_RE.search(source) and CHAT_CONTEXT_RE.search(source) + ) + if ( + endpoint_match is None + and method_match is None + and not ((telegram_named and telegram_context) or token_and_chat_context) + ): + continue + + method = ( + endpoint_match.group("method") + if endpoint_match is not None + else method_match.group("method") + if method_match is not None + else "dynamic" + ) + line_number = ( + resolved_python_transports[0][0] + if resolved_python_transports + else start_line + source[: transport_match.start()].count("\n") + ) + detection_kind = ( + "direct_bot_api_endpoint" + if endpoint_match is not None + else "custom_direct_sender" + ) + key = (line_number, method.lower(), detection_kind) + if key in seen: + continue + seen.add(key) + source_line = ( + source_lines[line_number - 1] if line_number <= len(source_lines) else "" + ) + findings.append( + { + "line": line_number, + "method": method, + "detection_kind": detection_kind, + "function": function_name, + "function_qualified": function_qualified, + "sanitized_excerpt": sanitize_excerpt(source_line), + } + ) + return findings + + def surface_kind(relative_path: str) -> str: + if relative_path.startswith(".github/workflows/"): + return "github_frozen_legacy_workflow_direct_bot_api" if relative_path.startswith(".gitea/workflows/"): return "gitea_workflow_direct_bot_api" if relative_path.startswith("scripts/ops/"): @@ -186,6 +672,12 @@ def surface_kind(relative_path: str) -> str: return "other_direct_bot_api" +def source_truth_classification(relative_path: str) -> str: + if relative_path.startswith(".github/workflows/"): + return "frozen_legacy_source_truth" + return "active_repo_source_truth" + + def line_hash(relative_path: str, line_number: int, line: str) -> str: payload = f"{relative_path}:{line_number}:{line.strip()}".encode("utf-8") return hashlib.sha256(payload).hexdigest()[:16] @@ -200,42 +692,50 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: for path in files: relative_path = path.relative_to(root).as_posix() text = path.read_text(encoding="utf-8", errors="replace") - for line_number, line in enumerate(text.splitlines(), start=1): - direct_match = DIRECT_BOT_API_RE.search(line) - if direct_match: - kind = surface_kind(relative_path) - direct_calls.append( - { - "egress_surface_id": f"telegram_egress:{kind}:{relative_path}:{line_number}", - "surface_kind": kind, - "path": relative_path, - "line": line_number, - "line_hash": line_hash(relative_path, line_number, line), - "method": direct_match.group("method"), - "sanitized_excerpt": sanitize_excerpt(line), - "required_owner_fields": REQUIRED_OWNER_FIELDS, - "reviewer_checks": REVIEWER_CHECKS, - "outcome_lanes": OUTCOME_LANES, - "blocked_actions": BLOCKED_ACTIONS, - "owner_response_received": False, - "owner_response_accepted": False, - "formatter_convergence_accepted": False, - "redaction_contract_accepted": False, - "delivery_receipt_accepted": False, - "direct_bot_api_migration_authorized": False, - "telegram_send_authorized": False, - "bot_api_call_authorized": False, - "workflow_modification_authorized": False, - "script_modification_authorized": False, - "secret_value_collection_allowed": False, - "raw_payload_storage_allowed": False, - "production_write_authorized": False, - "runtime_gate": False, - "action_buttons_allowed": False, - "not_authorization": True, - } - ) + text_lines = text.splitlines() + for finding in scan_direct_bot_api_surfaces(relative_path, text): + line_number = int(finding["line"]) + line = text_lines[line_number - 1] if line_number <= len(text_lines) else "" + kind = surface_kind(relative_path) + truth_classification = source_truth_classification(relative_path) + direct_calls.append( + { + "egress_surface_id": f"telegram_egress:{kind}:{relative_path}:{line_number}", + "surface_kind": kind, + "source_truth_classification": truth_classification, + "path": relative_path, + "line": line_number, + "line_hash": line_hash(relative_path, line_number, line), + "method": finding["method"], + "detection_kind": finding["detection_kind"], + "function": finding["function"], + "sanitized_excerpt": finding["sanitized_excerpt"], + "required_owner_fields": REQUIRED_OWNER_FIELDS, + "reviewer_checks": REVIEWER_CHECKS, + "outcome_lanes": OUTCOME_LANES, + "blocked_actions": BLOCKED_ACTIONS, + "owner_response_received": False, + "owner_response_accepted": False, + "formatter_convergence_accepted": False, + "redaction_contract_accepted": False, + "delivery_receipt_accepted": False, + "direct_bot_api_migration_authorized": False, + "telegram_send_authorized": False, + "bot_api_call_authorized": False, + "workflow_modification_authorized": False, + "script_modification_authorized": False, + "secret_value_collection_allowed": False, + "raw_payload_storage_allowed": False, + "production_write_authorized": False, + "runtime_execution_authorized": False, + "workflow_execution_authorized": False, + "runtime_gate": False, + "action_buttons_allowed": False, + "not_authorization": True, + } + ) + for line_number, line in enumerate(text_lines, start=1): if GATEWAY_CALLSITE_RE.search(line): gateway_calls.append( { @@ -245,14 +745,56 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: } ) + active_direct_calls = [ + item + for item in direct_calls + if item["source_truth_classification"] == "active_repo_source_truth" + ] + frozen_legacy_direct_calls = [ + item + for item in direct_calls + if item["source_truth_classification"] == "frozen_legacy_source_truth" + ] direct_files = sorted({item["path"] for item in direct_calls}) - workflow_direct_calls = [item for item in direct_calls if item["surface_kind"] == "gitea_workflow_direct_bot_api"] - ops_direct_calls = [item for item in direct_calls if item["surface_kind"] == "ops_script_direct_bot_api"] - ci_direct_calls = [item for item in direct_calls if item["surface_kind"] == "ci_script_direct_bot_api"] - api_direct_calls = [item for item in direct_calls if item["surface_kind"] == "api_direct_bot_api"] + active_direct_files = sorted({item["path"] for item in active_direct_calls}) + frozen_legacy_direct_files = sorted( + {item["path"] for item in frozen_legacy_direct_calls} + ) + workflow_direct_calls = [ + item + for item in direct_calls + if item["surface_kind"] == "gitea_workflow_direct_bot_api" + ] + ops_direct_calls = [ + item + for item in direct_calls + if item["surface_kind"] == "ops_script_direct_bot_api" + ] + ci_direct_calls = [ + item + for item in direct_calls + if item["surface_kind"] == "ci_script_direct_bot_api" + ] + api_direct_calls = [ + item for item in direct_calls if item["surface_kind"] == "api_direct_bot_api" + ] + custom_direct_senders = [ + item + for item in direct_calls + if item["detection_kind"] == "custom_direct_sender" + ] + direct_endpoints = [ + item + for item in direct_calls + if item["detection_kind"] == "direct_bot_api_endpoint" + ] telegram_gateway_path = root / "apps/api/src/services/telegram_gateway.py" - telegram_gateway_text = telegram_gateway_path.read_text(encoding="utf-8", errors="replace") - gateway_formatter_present = "normalize_telegram_send_message_payload" in telegram_gateway_text + telegram_gateway_text = telegram_gateway_path.read_text( + encoding="utf-8", errors="replace" + ) + gateway_formatter_present = ( + "normalize_telegram_send_message_payload" in telegram_gateway_text + ) return { "schema_version": "telegram_notification_egress_inventory_v1", @@ -265,12 +807,24 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: "scanned_file_count": len(files), "direct_bot_api_file_count": len(direct_files), "direct_bot_api_call_count": len(direct_calls), + "active_direct_bot_api_file_count": len(active_direct_files), + "active_direct_bot_api_call_count": len(active_direct_calls), + "github_frozen_legacy_direct_bot_api_file_count": len( + frozen_legacy_direct_files + ), + "github_frozen_legacy_direct_bot_api_call_count": len( + frozen_legacy_direct_calls + ), + "direct_bot_api_endpoint_count": len(direct_endpoints), + "custom_direct_sender_count": len(custom_direct_senders), "workflow_direct_bot_api_call_count": len(workflow_direct_calls), "ops_script_direct_bot_api_call_count": len(ops_direct_calls), "ci_script_direct_bot_api_call_count": len(ci_direct_calls), "api_direct_bot_api_call_count": len(api_direct_calls), "gateway_normalized_callsite_count": len(gateway_calls), - "gateway_final_exit_formatter_present_count": 1 if gateway_formatter_present else 0, + "gateway_final_exit_formatter_present_count": 1 + if gateway_formatter_present + else 0, "required_owner_field_count": len(REQUIRED_OWNER_FIELDS), "reviewer_check_count": len(REVIEWER_CHECKS), "outcome_lane_count": len(OUTCOME_LANES), @@ -284,6 +838,7 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: "telegram_send_authorized_count": 0, "bot_api_call_authorized_count": 0, "workflow_modification_authorized_count": 0, + "workflow_execution_authorized_count": 0, "script_modification_authorized_count": 0, "secret_value_collection_allowed_count": 0, "raw_payload_storage_allowed_count": 0, @@ -296,6 +851,8 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: "telegram_send_authorized": False, "bot_api_call_authorized": False, "workflow_modification_authorized": False, + "workflow_execution_authorized": False, + "github_workflow_execution_authorized": False, "script_modification_authorized": False, "secret_value_collection_allowed": False, "secret_hash_collection_allowed": False, @@ -311,7 +868,8 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: "direct_bot_api_calls": direct_calls, "gateway_normalized_callsite_refs": gateway_calls, "operator_interpretation": [ - "direct_bot_api_call_count 大於 0 代表仍有 workflow / ops / API 旁路可能繞過 TelegramGateway formatter。", + "active_direct_bot_api_call_count 大於 0 代表仍有 active workflow / ops / API 旁路可能繞過 TelegramGateway formatter。", + ".github/workflows 僅列為 frozen_legacy_source_truth 供盤點，runtime_execution_authorized=false、workflow_execution_authorized=false，禁止執行。", "本清冊只建立 metadata-only egress surface，不送 Telegram、不修改 workflow / script、不讀 secret value。", "後續要收斂 direct Bot API 必須另走 owner response、formatter convergence、redaction contract、delivery receipt 與維護窗口。", ], @@ -321,11 +879,15 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: def validate(root: Path) -> None: report = build_report(root) if report["summary"]["gateway_final_exit_formatter_present_count"] != 1: - raise SystemExit("BLOCKED telegram egress inventory: gateway formatter not found") + raise SystemExit( + "BLOCKED telegram egress inventory: gateway formatter not found" + ) def main() -> None: - parser = argparse.ArgumentParser(description="Build Telegram notification egress inventory") + parser = argparse.ArgumentParser( + description="Build Telegram notification egress inventory" + ) parser.add_argument("--root", default=".", help="repository root") parser.add_argument("--output", help="write JSON snapshot") parser.add_argument("--generated-at", help="fixed generated_at timestamp") @@ -342,6 +904,9 @@ def main() -> None: print( "TELEGRAM_NOTIFICATION_EGRESS_INVENTORY_OK " f"direct_calls={report['summary']['direct_bot_api_call_count']} " + f"active={report['summary']['active_direct_bot_api_call_count']} " + f"frozen_legacy={report['summary']['github_frozen_legacy_direct_bot_api_call_count']} " + f"custom={report['summary']['custom_direct_sender_count']} " f"files={report['summary']['direct_bot_api_file_count']} " f"workflow={report['summary']['workflow_direct_bot_api_call_count']} " f"ops={report['summary']['ops_script_direct_bot_api_call_count']} " diff --git a/scripts/security/telegram-notification-egress-no-new-bypass-guard.py b/scripts/security/telegram-notification-egress-no-new-bypass-guard.py index bc2ebc4bc..098914e32 100644 --- a/scripts/security/telegram-notification-egress-no-new-bypass-guard.py +++ b/scripts/security/telegram-notification-egress-no-new-bypass-guard.py @@ -2,14 +2,16 @@ """檢查 Telegram 通知出口不可新增未登記 direct Bot API 旁路。 本 guard 只掃描 repo 原始碼與 committed snapshot，不讀 secret、不呼叫 -Telegram、不修改 workflow / script / API sender。既有 direct send 仍是待 -owner response 的基線；任何新增或變形的 direct Bot API endpoint 都必須先 -進 inventory / owner request / migration plan，而不是直接合併。 +Telegram、不修改或執行 workflow / script / API sender。Committed baseline +僅供 drift 比對；目前 active source 的 direct endpoint 或 custom sender 必須 +真正為零。Local ``.github/workflows`` 只以 frozen legacy source truth 稽核， +不得以 regex 漏掃、歷史基線或 frozen source 產生 false-green。 """ from __future__ import annotations import argparse +import ast import json import re import subprocess @@ -22,10 +24,13 @@ from typing import Any TAIPEI = timezone(timedelta(hours=8)) -SOURCE_SNAPSHOT = Path("docs/security/telegram-notification-egress-inventory.snapshot.json") +SOURCE_SNAPSHOT = Path( + "docs/security/telegram-notification-egress-inventory.snapshot.json" +) SCAN_ROOTS = ( Path("agent99-control-plane.ps1"), Path(".gitea/workflows"), + Path(".github/workflows"), Path("scripts/ops"), Path("scripts/ci"), Path("scripts/reboot-recovery"), @@ -50,6 +55,65 @@ BOT_ENDPOINT_RE = re.compile( + r")\b", re.IGNORECASE, ) +GUARDED_BOT_METHOD_RE = re.compile( + r"\b(?P" + + "|".join(re.escape(method) for method in GUARDED_BOT_METHODS) + + r")\b", + re.IGNORECASE, +) +COMPACT_BOT_ENDPOINT_RE = re.compile( + r"api\.telegram\.org.{0,1024}?bot.{0,512}?/(?P" + + "|".join(re.escape(method) for method in GUARDED_BOT_METHODS) + + r")", + re.IGNORECASE, +) +DIRECT_HTTP_TRANSPORT_RE = re.compile( + r"(?:" + r"requests\s*\.\s*Session\s*\([^)]*\)\s*\.\s*(?:post|request|send)\s*\(|" + r"requests\s*\.\s*(?:post|request)\s*\(|" + r"httpx\s*\.\s*(?:AsyncClient|Client)\s*\([^)]*\)\s*\.\s*(?:post|request|send)\s*\(|" + r"httpx\s*\.\s*(?:post|request)\s*\(|" + r"(?:urllib\s*\.\s*request\s*\.\s*)?build_opener\s*\([^)]*\)\s*\.\s*open\s*\(|" + r"urllib\s*\.\s*request\s*\.\s*(?:urlopen|Request)\s*\(|" + r"\burlopen\s*\(|" + r"\b(?:_?opener)\s*\.\s*open\s*\(|" + r"\bInvoke-(?:WebRequest|RestMethod)\b|" + r"\.\s*PostAsync\s*\(|" + r"\.\s*Upload(?:String|Data|File)\s*\(|" + r"\b(?:fetch|curl|wget)\b|" + r"\b(?:_http_client|client|session)\s*\.\s*(?:post|request|send)\s*\(" + r")", + re.IGNORECASE, +) +TELEGRAM_CONTEXT_RE = re.compile( + r"(?:telegram|bot[_-]?token|chat[_-]?id)", re.IGNORECASE +) +BOT_TOKEN_CONTEXT_RE = re.compile( + r"(?:\bbot[_-]?token\b|\btelegram[_-]?(?:bot[_-]?)?token\b|\$Token\b)", + re.IGNORECASE, +) +CHAT_CONTEXT_RE = re.compile( + r"(?:\bchat[_-]?id\b|\$ChatId\b)", + re.IGNORECASE, +) +POWERSHELL_FUNCTION_RE = re.compile( + r"^function\s+(?P[A-Za-z0-9_-]+)\s*\{", re.IGNORECASE | re.MULTILINE +) +CANONICAL_FINAL_EXITS = { + ( + "apps/api/src/services/telegram_gateway.py", + "TelegramGateway._send_request", + ), +} +_AST_IMPORT_NODE_TYPES = (ast.Import, ast.ImportFrom) +_AST_FUNCTION_NODE_TYPES = (ast.FunctionDef, ast.AsyncFunctionDef) +_AST_SEQUENCE_TARGET_NODE_TYPES = (ast.Tuple, ast.List) +_AST_NON_MODULE_SCOPE_NODE_TYPES = ( + ast.ClassDef, + ast.FunctionDef, + ast.AsyncFunctionDef, + ast.Lambda, +) SECRET_INTERPOLATION_RE = re.compile(r"\$\{\{\s*secrets\.[^}]+\}\}") BOT_TOKEN_URL_RE = re.compile( r"api\.telegram\.org/bot.*?/(?P" @@ -57,6 +121,12 @@ BOT_TOKEN_URL_RE = re.compile( + r")\b", re.IGNORECASE, ) +BOT_TOKEN_FRAGMENT_RE = re.compile( + r"bot[^/\s\"']+/(?P" + + "|".join(re.escape(method) for method in GUARDED_BOT_METHODS) + + r")\b", + re.IGNORECASE, +) def git_short_sha(root: Path) -> str: @@ -96,13 +166,436 @@ def sanitize_excerpt(line: str) -> str: lambda match: f"api.telegram.org/bot/{match.group('method')}", excerpt, ) + excerpt = BOT_TOKEN_FRAGMENT_RE.sub( + lambda match: f"bot/{match.group('method')}", + excerpt, + ) return excerpt[:180] +def _compact_source(source: str) -> str: + without_string_prefixes = re.sub( + r"(?i)\b(?:fr|rf|f|r|u|b)(?=[\"'])", + "", + source, + ) + return re.sub(r"[\s\"'`+()]+", "", without_string_prefixes) + + +_PYTHON_HTTP_TRANSPORT_CALL_RE = re.compile( + r"^(?:" + r"requests\.(?:post|request)|" + r"requests\.Session\(\)\.(?:post|request|send)|" + r"httpx\.(?:post|request)|" + r"httpx\.(?:AsyncClient|Client)\(\)\.(?:post|request|send)|" + r"urllib\.request\.(?:urlopen|Request)|" + r"urllib\.request\.build_opener\(\)\.open" + r")$" +) + + +def _record_python_import_alias( + node: ast.AST, + aliases: dict[str, str], +) -> None: + if isinstance(node, ast.Import): + for imported in node.names: + bound_name = imported.asname or imported.name.split(".", 1)[0] + aliases[bound_name] = ( + imported.name if imported.asname else bound_name + ) + return + + module = str(node.module or "").strip(".") + if not module: + return + for imported in node.names: + if imported.name == "*": + continue + aliases[imported.asname or imported.name] = ( + f"{module}.{imported.name}" + ) + + +def _resolve_python_expression( + node: ast.AST, + aliases: dict[str, str], +) -> str | None: + if isinstance(node, ast.Name): + return aliases.get(node.id, node.id) + if isinstance(node, ast.Attribute): + owner = _resolve_python_expression(node.value, aliases) + return f"{owner}.{node.attr}" if owner else None + if isinstance(node, ast.Call): + callable_name = _resolve_python_expression(node.func, aliases) + return f"{callable_name}()" if callable_name else None + return None + + +def _assignment_target_names(node: ast.AST) -> list[str]: + if isinstance(node, ast.Name): + return [node.id] + if isinstance(node, _AST_SEQUENCE_TARGET_NODE_TYPES): + return [ + name + for child in node.elts + for name in _assignment_target_names(child) + ] + return [] + + +def _python_parent_map(tree: ast.AST) -> dict[ast.AST, ast.AST]: + return { + child: parent + for parent in ast.walk(tree) + for child in ast.iter_child_nodes(parent) + } + + +def _python_function_qualified_name( + function: ast.FunctionDef | ast.AsyncFunctionDef, + parents: dict[ast.AST, ast.AST], +) -> str: + parts = [function.name] + parent = parents.get(function) + while parent is not None: + if isinstance(parent, (ast.ClassDef, *_AST_FUNCTION_NODE_TYPES)): + parts.append(parent.name) + parent = parents.get(parent) + return ".".join(reversed(parts)) + + +def _nearest_python_function( + node: ast.AST, + parents: dict[ast.AST, ast.AST], +) -> ast.FunctionDef | ast.AsyncFunctionDef | None: + parent = parents.get(node) + while parent is not None: + if isinstance(parent, _AST_FUNCTION_NODE_TYPES): + return parent + parent = parents.get(parent) + return None + + +def _is_python_module_scope( + node: ast.AST, + parents: dict[ast.AST, ast.AST], +) -> bool: + parent = parents.get(node) + while parent is not None: + if isinstance(parent, _AST_NON_MODULE_SCOPE_NODE_TYPES): + return False + parent = parents.get(parent) + return True + + +def _resolve_python_transport_calls( + nodes: list[ast.AST], + aliases: dict[str, str], +) -> list[tuple[int, str]]: + resolved_calls: list[tuple[int, str]] = [] + for node in sorted( + nodes, + key=lambda item: ( + int(getattr(item, "lineno", 0)), + int(getattr(item, "col_offset", 0)), + 0 if isinstance(item, _AST_IMPORT_NODE_TYPES) else 1, + ), + ): + if isinstance(node, _AST_IMPORT_NODE_TYPES): + _record_python_import_alias(node, aliases) + continue + if isinstance(node, ast.Assign): + resolved_value = _resolve_python_expression(node.value, aliases) + for target in node.targets: + for name in _assignment_target_names(target): + if resolved_value: + aliases[name] = resolved_value + else: + aliases.pop(name, None) + continue + if isinstance(node, ast.AnnAssign): + resolved_value = ( + _resolve_python_expression(node.value, aliases) + if node.value is not None + else None + ) + for name in _assignment_target_names(node.target): + if resolved_value: + aliases[name] = resolved_value + else: + aliases.pop(name, None) + continue + if not isinstance(node, ast.Call): + continue + qualified_call = _resolve_python_expression(node.func, aliases) + if qualified_call and _PYTHON_HTTP_TRANSPORT_CALL_RE.fullmatch( + qualified_call + ): + resolved_calls.append((node.lineno, qualified_call)) + return sorted(set(resolved_calls)) + + +def _python_transport_calls_by_scope( + text: str, +) -> tuple[ + dict[tuple[str, int, int], list[tuple[int, str]]], + list[tuple[int, str]], +]: + """Resolve HTTP aliases in function and true module-level AST scopes.""" + try: + tree = ast.parse(text) + except SyntaxError: + return {}, [] + + parents = _python_parent_map(tree) + module_aliases: dict[str, str] = {} + module_nodes = [ + node + for node in ast.walk(tree) + if _is_python_module_scope(node, parents) + ] + module_calls = _resolve_python_transport_calls( + module_nodes, + module_aliases, + ) + + transports: dict[tuple[str, int, int], list[tuple[int, str]]] = {} + function_nodes = [ + node + for node in ast.walk(tree) + if isinstance(node, _AST_FUNCTION_NODE_TYPES) + and node.end_lineno is not None + ] + for function in function_nodes: + aliases = dict(module_aliases) + arguments = [ + *function.args.posonlyargs, + *function.args.args, + *function.args.kwonlyargs, + ] + if function.args.vararg is not None: + arguments.append(function.args.vararg) + if function.args.kwarg is not None: + arguments.append(function.args.kwarg) + for argument in arguments: + aliases.pop(argument.arg, None) + + scoped_nodes = [ + node + for node in ast.walk(function) + if node is function + or _nearest_python_function(node, parents) is function + ] + resolved_calls = _resolve_python_transport_calls(scoped_nodes, aliases) + + if resolved_calls: + qualified_name = _python_function_qualified_name(function, parents) + transports[ + ( + qualified_name, + function.lineno, + int(function.end_lineno), + ) + ] = resolved_calls + return transports, module_calls + + +def _transport_window_units( + text: str, + *, + excluded_line_ranges: list[tuple[int, int]], +) -> list[tuple[str, str, int, int, str]]: + lines = text.splitlines() + units: list[tuple[str, str, int, int, str]] = [] + for match in DIRECT_HTTP_TRANSPORT_RE.finditer(text): + line_number = text.count("\n", 0, match.start()) + 1 + if any(start <= line_number <= end for start, end in excluded_line_ranges): + continue + start_line = max(1, line_number - 20) + end_line = min(len(lines), line_number + 20) + units.append( + ( + "", + "", + start_line, + end_line, + "\n".join(lines[start_line - 1 : end_line]), + ) + ) + return units + + +def _source_units( + relative_path: str, + text: str, +) -> list[tuple[str, str, int, int, str]]: + lines = text.splitlines() + units: list[tuple[str, str, int, int, str]] = [] + ranges: list[tuple[int, int]] = [] + if relative_path.endswith(".py"): + try: + tree = ast.parse(text) + except SyntaxError: + tree = None + if tree is not None: + parents = _python_parent_map(tree) + nodes = [ + node + for node in ast.walk(tree) + if isinstance(node, _AST_FUNCTION_NODE_TYPES) + and getattr(node, "end_lineno", None) + ] + for node in sorted( + nodes, key=lambda item: (item.lineno, item.end_lineno or item.lineno) + ): + end_line = int(node.end_lineno or node.lineno) + ranges.append((node.lineno, end_line)) + units.append( + ( + node.name, + _python_function_qualified_name(node, parents), + node.lineno, + end_line, + "\n".join(lines[node.lineno - 1 : end_line]), + ) + ) + elif relative_path.endswith(".ps1"): + matches = list(POWERSHELL_FUNCTION_RE.finditer(text)) + for index, match in enumerate(matches): + start_line = text.count("\n", 0, match.start()) + 1 + end_offset = ( + matches[index + 1].start() if index + 1 < len(matches) else len(text) + ) + end_line = text.count("\n", 0, end_offset) + 1 + ranges.append((start_line, end_line)) + units.append( + ( + match.group("name"), + match.group("name"), + start_line, + end_line, + text[match.start() : end_offset], + ) + ) + return units + _transport_window_units(text, excluded_line_ranges=ranges) + + +def scan_direct_bot_api_surfaces( + relative_path: str, + text: str, +) -> list[dict[str, Any]]: + """Find direct Telegram transports, including split URLs and custom senders.""" + + source_lines = text.splitlines() + findings: list[dict[str, Any]] = [] + seen: set[tuple[int, str, str]] = set() + python_transports, module_transports = ( + _python_transport_calls_by_scope(text) + if relative_path.endswith(".py") + else ({}, []) + ) + source_units = _source_units(relative_path, text) + for line_number, _qualified_call in module_transports: + start_line = max(1, line_number - 20) + end_line = min(len(source_lines), line_number + 20) + source_units.append( + ( + "", + "", + start_line, + end_line, + "\n".join(source_lines[start_line - 1 : end_line]), + ) + ) + + for ( + function_name, + function_qualified, + start_line, + _end_line, + source, + ) in source_units: + if (relative_path, function_qualified) in CANONICAL_FINAL_EXITS: + continue + resolved_python_transports = ( + [ + item + for item in module_transports + if start_line <= item[0] <= _end_line + ] + if function_qualified == "" + else python_transports.get( + (function_qualified, start_line, _end_line), + [], + ) + ) + transport_match = DIRECT_HTTP_TRANSPORT_RE.search(source) + if transport_match is None and not resolved_python_transports: + continue + method_match = GUARDED_BOT_METHOD_RE.search(source) + endpoint_match = COMPACT_BOT_ENDPOINT_RE.search(_compact_source(source)) + telegram_named = bool( + re.search(r"(?:telegram|bot)", function_name, re.IGNORECASE) + ) + telegram_context = bool(TELEGRAM_CONTEXT_RE.search(source)) + token_and_chat_context = bool( + BOT_TOKEN_CONTEXT_RE.search(source) and CHAT_CONTEXT_RE.search(source) + ) + if ( + endpoint_match is None + and method_match is None + and not ((telegram_named and telegram_context) or token_and_chat_context) + ): + continue + + method = ( + endpoint_match.group("method") + if endpoint_match is not None + else method_match.group("method") + if method_match is not None + else "dynamic" + ) + line_number = ( + resolved_python_transports[0][0] + if resolved_python_transports + else start_line + source[: transport_match.start()].count("\n") + ) + detection_kind = ( + "direct_bot_api_endpoint" + if endpoint_match is not None + else "custom_direct_sender" + ) + key = (line_number, method.lower(), detection_kind) + if key in seen: + continue + seen.add(key) + source_line = ( + source_lines[line_number - 1] if line_number <= len(source_lines) else "" + ) + findings.append( + { + "line": line_number, + "method": method, + "detection_kind": detection_kind, + "function": function_name, + "function_qualified": function_qualified, + "sanitized_excerpt": sanitize_excerpt(source_line), + } + ) + return findings + + def signature(path: str, method: str, sanitized_excerpt: str) -> str: return f"{path}::{method.lower()}::{sanitized_excerpt}" +def source_truth_classification(relative_path: str) -> str: + if relative_path.startswith(".github/workflows/"): + return "frozen_legacy_source_truth" + return "active_repo_source_truth" + + def load_source_snapshot(root: Path) -> dict[str, Any]: snapshot_path = root / SOURCE_SNAPSHOT return json.loads(snapshot_path.read_text(encoding="utf-8")) @@ -111,9 +604,16 @@ def load_source_snapshot(root: Path) -> dict[str, Any]: def build_baseline(source_snapshot: dict[str, Any]) -> Counter[str]: baseline: Counter[str] = Counter() for item in source_snapshot.get("direct_bot_api_calls", []): + if ( + source_truth_classification(str(item["path"])) + == "frozen_legacy_source_truth" + ): + continue excerpt = item.get("sanitized_excerpt", "") match = BOT_ENDPOINT_RE.search(excerpt) - method = match.group("method") if match else "sendMessage" + method = str( + item.get("method") or (match.group("method") if match else "dynamic") + ) baseline[signature(item["path"], method, excerpt)] += 1 return baseline @@ -123,19 +623,25 @@ def scan_current_direct_endpoints(root: Path) -> list[dict[str, Any]]: for path in iter_scannable_files(root): relative_path = path.relative_to(root).as_posix() text = path.read_text(encoding="utf-8", errors="replace") - for line_number, line in enumerate(text.splitlines(), start=1): - for match in BOT_ENDPOINT_RE.finditer(line): - method = match.group("method") - sanitized = sanitize_excerpt(line) - findings.append( - { - "path": relative_path, - "line": line_number, - "method": method, - "sanitized_excerpt": sanitized, - "signature": signature(relative_path, method, sanitized), - } - ) + for finding in scan_direct_bot_api_surfaces(relative_path, text): + method = finding["method"] + sanitized = finding["sanitized_excerpt"] + findings.append( + { + "path": relative_path, + "line": finding["line"], + "method": method, + "detection_kind": finding["detection_kind"], + "function": finding["function"], + "sanitized_excerpt": sanitized, + "signature": signature(relative_path, method, sanitized), + "source_truth_classification": source_truth_classification( + relative_path + ), + "runtime_execution_authorized": False, + "workflow_execution_authorized": False, + } + ) return findings @@ -153,7 +659,17 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: generated = generated_at or datetime.now(TAIPEI).isoformat(timespec="seconds") source_snapshot = load_source_snapshot(root) baseline = build_baseline(source_snapshot) - current_findings = scan_current_direct_endpoints(root) + detected_findings = scan_current_direct_endpoints(root) + current_findings = [ + item + for item in detected_findings + if item["source_truth_classification"] == "active_repo_source_truth" + ] + frozen_legacy_findings = [ + item + for item in detected_findings + if item["source_truth_classification"] == "frozen_legacy_source_truth" + ] remaining_baseline = baseline.copy() new_bypass_findings: list[dict[str, Any]] = [] @@ -172,23 +688,48 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: current_files = sorted({item["path"] for item in current_findings}) new_bypass_files = sorted({item["path"] for item in new_bypass_findings}) counts_by_method = method_counts(current_findings) + custom_direct_senders = [ + item + for item in current_findings + if item["detection_kind"] == "custom_direct_sender" + ] + direct_endpoints = [ + item + for item in current_findings + if item["detection_kind"] == "direct_bot_api_endpoint" + ] source_summary = source_snapshot["summary"] return { "schema_version": "telegram_notification_egress_no_new_bypass_guard_v1", "generated_at": generated, "git_commit": git_short_sha(root), - "status": "pass_no_new_bypass" if not new_bypass_findings else "blocked_new_bypass_detected", + "status": "pass_no_direct_or_custom_bypass" + if not current_findings + else "blocked_direct_or_custom_bypass_detected", "mode": "repo_source_scan_no_secret_value_no_telegram_send", "source_snapshot": SOURCE_SNAPSHOT.as_posix(), "guarded_roots": [path.as_posix() for path in SCAN_ROOTS], "guarded_bot_methods": list(GUARDED_BOT_METHODS), "summary": { - "source_direct_bot_api_call_count": source_summary["direct_bot_api_call_count"], - "source_direct_bot_api_file_count": source_summary["direct_bot_api_file_count"], + "source_direct_bot_api_call_count": source_summary[ + "direct_bot_api_call_count" + ], + "source_direct_bot_api_file_count": source_summary[ + "direct_bot_api_file_count" + ], "baseline_signature_count": sum(baseline.values()), + "detected_direct_bot_api_call_count": len(detected_findings), "current_direct_bot_api_call_count": len(current_findings), "current_direct_bot_api_file_count": len(current_files), + "current_direct_bot_api_endpoint_count": len(direct_endpoints), + "current_custom_direct_sender_count": len(custom_direct_senders), + "github_frozen_legacy_direct_bot_api_call_count": len( + frozen_legacy_findings + ), + "github_frozen_legacy_direct_bot_api_file_count": len( + {item["path"] for item in frozen_legacy_findings} + ), "guarded_method_count": len(GUARDED_BOT_METHODS), "sendMessage_call_count": counts_by_method["sendMessage"], "sendDocument_call_count": counts_by_method["sendDocument"], @@ -209,7 +750,9 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: ), "new_bypass_count": len(new_bypass_findings), "new_bypass_file_count": len(new_bypass_files), - "removed_baseline_call_count": sum(item["removed_count"] for item in removed_baseline_signatures), + "removed_baseline_call_count": sum( + item["removed_count"] for item in removed_baseline_signatures + ), "runtime_gate_count": 0, "action_button_count": 0, }, @@ -218,6 +761,8 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: "telegram_send_authorized": False, "bot_api_call_authorized": False, "workflow_modification_authorized": False, + "workflow_execution_authorized": False, + "github_workflow_execution_authorized": False, "script_modification_authorized": False, "api_sender_refactor_authorized": False, "secret_value_collection_allowed": False, @@ -231,15 +776,18 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: "not_authorization": True, }, "current_direct_bot_api_calls": current_findings, + "detected_direct_bot_api_calls": detected_findings, + "frozen_legacy_direct_bot_api_calls": frozen_legacy_findings, "new_bypass_findings": new_bypass_findings, "removed_baseline_signatures": removed_baseline_signatures, "operator_interpretation": [ - "new_bypass_count 維持 0 才代表沒有新增未登記 Telegram Bot API 直送旁路。", + "current_direct_bot_api_call_count 必須為 0，才代表沒有 Telegram direct/custom bypass。", + ".github/workflows 僅以 frozen_legacy_source_truth 盤點，不計入 active/new failure；runtime 與 workflow execution 均未授權且禁止執行。", ( f"committed inventory 目前有 {source_summary['direct_bot_api_call_count']} 個 direct Bot API call site；" - "若數值大於 0 仍是待 controlled migration 的基線，不代表已批准保留。" + "baseline 只做 drift 比對，不能批准或隱藏目前 source 旁路。" ), - "sendDocument / sendPhoto / sendMediaGroup 等附件型出口若出現在 repo source，會被視為新增旁路並阻擋。", + "分段 URL、requests/httpx/urllib、Invoke-WebRequest/Invoke-RestMethod 與自訂 sender 都會被掃描。", "本 guard 只讀 repo source 與 committed snapshot，不送 Telegram、不讀 Bot token、不修改 workflow / script / API sender。", ], } @@ -248,10 +796,11 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: def validate(root: Path) -> None: report = build_report(root) errors: list[str] = [] - if report["summary"]["new_bypass_count"]: - for item in report["new_bypass_findings"]: + if report["summary"]["current_direct_bot_api_call_count"]: + for item in report["current_direct_bot_api_calls"]: errors.append( - f"{item['path']}:{item['line']}: 新增未登記 Telegram Bot API 旁路 {item['method']}" + f"{item['path']}:{item['line']}: Telegram direct/custom bypass " + f"{item['detection_kind']} {item['method']} ({item['function']})" ) if errors: @@ -262,10 +811,14 @@ def validate(root: Path) -> None: def main() -> None: - parser = argparse.ArgumentParser(description="檢查 Telegram 通知出口不可新增未登記 direct Bot API 旁路") + parser = argparse.ArgumentParser( + description="檢查 Telegram 通知出口不可新增未登記 direct Bot API 旁路" + ) parser.add_argument("--root", default=".", help="repository root") parser.add_argument("--output", help="寫出 JSON 報告") - parser.add_argument("--generated-at", help="固定 generated_at 時間，供 committed snapshot 使用") + parser.add_argument( + "--generated-at", help="固定 generated_at 時間，供 committed snapshot 使用" + ) args = parser.parse_args() root = Path(args.root).resolve() @@ -273,13 +826,18 @@ def main() -> None: if args.output: output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + output.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) validate(root) summary = report["summary"] print( "TELEGRAM_NOTIFICATION_EGRESS_NO_NEW_BYPASS_GUARD_OK " f"current={summary['current_direct_bot_api_call_count']} " + f"custom={summary['current_custom_direct_sender_count']} " + f"frozen_legacy={summary['github_frozen_legacy_direct_bot_api_call_count']} " f"baseline={summary['baseline_signature_count']} " f"new={summary['new_bypass_count']} " f"sendDocument={summary['sendDocument_call_count']} " From 730b6e51724227ec44da8747233a6704229bb1ea Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 22:06:40 +0800 Subject: [PATCH 6/7] test(telegram): require verified delivery receipts --- .../tests/test_ansible_verified_closure.py | 13 ++++++++- .../test_channel_hub_grouped_alert_events.py | 27 +++++++++++++++++-- .../TELEGRAM-CANONICAL-ROUTING-INVENTORY.md | 2 +- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/apps/api/tests/test_ansible_verified_closure.py b/apps/api/tests/test_ansible_verified_closure.py index f43b2eff5..966887f36 100644 --- a/apps/api/tests/test_ansible_verified_closure.py +++ b/apps/api/tests/test_ansible_verified_closure.py @@ -744,7 +744,13 @@ async def test_controlled_result_pending_reservation_never_resends_provider( return None def json(self) -> dict: - return {"ok": True, "result": {"message_id": 456}} + return { + "ok": True, + "result": { + "message_id": 456, + "chat": {"id": "sre-chat"}, + }, + } class _Client: def __init__(self) -> None: @@ -849,6 +855,11 @@ async def test_no_write_result_reuses_durable_suppression_without_provider_send( monkeypatch: pytest.MonkeyPatch, ) -> None: gateway = TelegramGateway() + monkeypatch.setattr( + telegram_gateway_module.settings, + "SRE_GROUP_CHAT_ID", + "sre-chat", + ) gateway._initialized = True client = AsyncMock() gateway._http_client = client diff --git a/apps/api/tests/test_channel_hub_grouped_alert_events.py b/apps/api/tests/test_channel_hub_grouped_alert_events.py index 17d10478e..d7f628ede 100644 --- a/apps/api/tests/test_channel_hub_grouped_alert_events.py +++ b/apps/api/tests/test_channel_hub_grouped_alert_events.py @@ -1,7 +1,7 @@ from __future__ import annotations -import json import inspect +import json from uuid import uuid4 from src.services.channel_hub import ( @@ -20,6 +20,7 @@ from src.services.channel_hub import ( mirror_inbound_event, record_outbound_message, ) +from src.services.telegram_gateway import _telegram_destination_binding _FAKE_SECRET_SHAPED_VALUE = "1234567890:" + "abcdefghijklmnopqrstuvwxyzABCDEFGH" @@ -54,11 +55,33 @@ async def test_send_telegram_interim_uses_gateway_mirror_path(monkeypatch) -> No "inbound_message_id": inbound_message_id, } ) + destination_binding = _telegram_destination_binding(chat_id) + destination_alias = "verified_inbound_chat" return { "ok": True, - "result": {"message_id": 42}, + "result": { + "message_id": 42, + "chat": {"id": chat_id}, + }, "_awooop_delivery_status": "sent", "_awooop_provider_send_performed": True, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "provider_send_performed": True, + "sender_bot_alias": "tsenyang_bot", + "destination_alias": destination_alias, + "destination_binding": destination_binding, + }, + "_awoooi_delivery_context": { + "schema_version": "telegram_delivery_context_v1", + "sender_bot_alias": "tsenyang_bot", + "destination_alias": destination_alias, + "destination_binding": destination_binding, + "payload_destination_binding": destination_binding, + "provider_destination_binding": destination_binding, + "destination_binding_verified": True, + }, } monkeypatch.setattr( diff --git a/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md index 2e30d028a..53b3a781b 100644 --- a/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md +++ b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md @@ -1,6 +1,6 @@ # Telegram 監控告警路由總帳 -> 狀態：`source_policy_ready_runtime_not_applied` +> 狀態：`source_policy_ready_runtime_not_applied` > 真相邊界：本表把「已證明現況」、「建議目的地」、「目前阻擋」、「source risk」與「receipt 強度」分開。建議不等於 live route；alias 不是 numeric chat ID；本變更沒有送 Telegram、讀 token/chat ID、改 runtime route 或部署。 ## 判讀規則 From ff8d9172475182ffe79050eb0f3240c112675571 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 00:42:15 +0800 Subject: [PATCH 7/7] fix(telegram): bind product alerts to canonical routes --- .gitea/workflows/agent-market-watch.yaml | 2 - .gitea/workflows/cd-dev.yaml | 1 - .gitea/workflows/cd.yaml | 1 - .gitea/workflows/code-review.yaml | 5 - .gitea/workflows/deploy-alerts.yaml | 3 - .gitea/workflows/e2e-health.yaml | 1 - .gitea/workflows/run-migration.yml | 3 - apps/api/src/api/v1/sentry_webhook.py | 27 ++- apps/api/src/api/v1/signoz_webhook.py | 9 +- apps/api/src/api/v1/webhooks.py | 51 +++++- apps/api/src/services/notification_matrix.py | 56 +++++++ apps/api/src/services/telegram_gateway.py | 16 +- .../test_alertmanager_webhook_metrics.py | 35 ++++ ...est_telegram_canonical_routing_registry.py | 155 ++++++++++++++++++ .../test_telegram_canonical_sender_gate.py | 27 +++ ...test_telegram_delivery_truth_gateway_v2.py | 7 +- docker-compose.yml | 4 +- .../TELEGRAM-CANONICAL-ROUTING-INVENTORY.md | 34 ++++ k8s/awoooi-prod/04-configmap.yaml | 1 - k8s/monitoring/k3s-alerts-supplemental.yaml | 3 + ops/monitoring/alerts-unified.yml | 18 ++ ops/monitoring/alerts.yml | 18 ++ scripts/ops/deploy-docker-health-monitor.sh | 2 +- scripts/ops/docker-health-monitor.sh | 2 +- 24 files changed, 450 insertions(+), 31 deletions(-) diff --git a/.gitea/workflows/agent-market-watch.yaml b/.gitea/workflows/agent-market-watch.yaml index 3430b73dd..e59021f3f 100644 --- a/.gitea/workflows/agent-market-watch.yaml +++ b/.gitea/workflows/agent-market-watch.yaml @@ -14,7 +14,6 @@ on: env: GITEA_ACTIONS_URL: http://192.168.0.110:3001/wooo/awoooi/actions - SRE_GROUP_CHAT_ID: "-1003711974679" jobs: market-watch: @@ -502,7 +501,6 @@ jobs: - name: Summarize actionable change or failure if: always() env: - TG_CHAT_ID: ${{ env.SRE_GROUP_CHAT_ID }} JOB_STATUS: ${{ job.status }} CANDIDATE_COUNT: ${{ steps.watch.outputs.candidate_count }} SOURCE_COUNT: ${{ steps.watch.outputs.source_count }} diff --git a/.gitea/workflows/cd-dev.yaml b/.gitea/workflows/cd-dev.yaml index 4ed76b4e6..4402d3a27 100644 --- a/.gitea/workflows/cd-dev.yaml +++ b/.gitea/workflows/cd-dev.yaml @@ -19,7 +19,6 @@ concurrency: env: HARBOR: 192.168.0.110:5000 HARBOR_MIRROR: 192.168.0.110:5001 - SRE_GROUP_CHAT_ID: "-1003711974679" OTEL_EXPORTER_OTLP_ENDPOINT: http://192.168.0.188:24318 OTEL_SERVICE_NAME: awoooi-cd-dev OTEL_RESOURCE_ATTRIBUTES: service.version=${{ github.sha }},deployment.environment=dev diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 5e610631c..6205e9158 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -32,7 +32,6 @@ concurrency: env: HARBOR: registry.wooo.work - SRE_GROUP_CHAT_ID: "-1003711974679" # Harbor Proxy Cache (指向 DockerHub 的內部 Mirror，避免拉取限額) HARBOR_MIRROR: 192.168.0.110:5001 # OTEL CI/CD 監控 (2026-03-31 #46c - 遷移到 Gitea) diff --git a/.gitea/workflows/code-review.yaml b/.gitea/workflows/code-review.yaml index 43be1c441..42f9a3990 100644 --- a/.gitea/workflows/code-review.yaml +++ b/.gitea/workflows/code-review.yaml @@ -13,7 +13,6 @@ concurrency: env: REPORT_URL: https://mo.wooo.work/code-review/ GITEA_ACTIONS_URL: http://192.168.0.110:3001/wooo/awoooi/actions - SRE_GROUP_CHAT_ID: "-1003711974679" jobs: ai-code-review: @@ -110,12 +109,10 @@ jobs: - name: Notify Code Review Start if: steps.stale.outputs.skip != 'true' env: - SRE_GROUP_CHAT_ID: ${{ env.SRE_GROUP_CHAT_ID }} SHORT_SHA: ${{ steps.ctx.outputs.short_sha }} BRANCH: ${{ steps.ctx.outputs.branch }} COMMIT_MSG: ${{ steps.ctx.outputs.commit_msg }} FILES_DISPLAY: ${{ steps.ctx.outputs.files_display }} - TG_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} run: | set -euo pipefail html_escape() { sed 's/&/\&/g; s//\>/g'; } @@ -151,9 +148,7 @@ jobs: - name: Notify Code Review Completion if: always() && steps.stale.outputs.skip != 'true' env: - SRE_GROUP_CHAT_ID: ${{ env.SRE_GROUP_CHAT_ID }} SHORT_SHA: ${{ steps.ctx.outputs.short_sha }} - TG_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} run: | set -euo pipefail REPORT=/tmp/code-review-report.json diff --git a/.gitea/workflows/deploy-alerts.yaml b/.gitea/workflows/deploy-alerts.yaml index a9904cfe9..0d1e366e1 100644 --- a/.gitea/workflows/deploy-alerts.yaml +++ b/.gitea/workflows/deploy-alerts.yaml @@ -12,9 +12,6 @@ on: # push path filtering 不得在 runner 搬遷 / 硬限流前重開。 workflow_dispatch: -env: - SRE_GROUP_CHAT_ID: "-1003711974679" - jobs: deploy-alerts: name: "Deploy Prometheus Alert Rules" diff --git a/.gitea/workflows/e2e-health.yaml b/.gitea/workflows/e2e-health.yaml index abb8d101a..1515cb75a 100644 --- a/.gitea/workflows/e2e-health.yaml +++ b/.gitea/workflows/e2e-health.yaml @@ -19,7 +19,6 @@ env: OTEL_EXPORTER_OTLP_ENDPOINT: http://192.168.0.188:24318 OTEL_SERVICE_NAME: awoooi-e2e OTEL_RESOURCE_ATTRIBUTES: deployment.environment=production - SRE_GROUP_CHAT_ID: "-1003711974679" jobs: e2e-health: diff --git a/.gitea/workflows/run-migration.yml b/.gitea/workflows/run-migration.yml index 872bffc53..cc978cf55 100644 --- a/.gitea/workflows/run-migration.yml +++ b/.gitea/workflows/run-migration.yml @@ -17,9 +17,6 @@ on: # 不得由 push 自動觸發；恢復前必須完成 runner 搬遷 / 硬限流與 migration dry-run gate。 workflow_dispatch: -env: - SRE_GROUP_CHAT_ID: "-1003711974679" - jobs: migrate: runs-on: awoooi-non110-ubuntu diff --git a/apps/api/src/api/v1/sentry_webhook.py b/apps/api/src/api/v1/sentry_webhook.py index 25494fd6e..1f3ff1ad0 100644 --- a/apps/api/src/api/v1/sentry_webhook.py +++ b/apps/api/src/api/v1/sentry_webhook.py @@ -39,6 +39,9 @@ from src.models.approval import ( from src.services.anomaly_counter import get_anomaly_counter from src.services.approval_db import get_approval_service from src.services.channel_hub import record_external_alert_event +from src.services.notification_matrix import ( + get_trusted_alert_canonical_route_context, +) from src.services.openclaw_http_service import get_openclaw_http_service from src.services.sentry_service import get_sentry_service @@ -648,6 +651,25 @@ async def send_sentry_telegram_alert( title = error_context.get("title", "Unknown Error") culprit = error_context.get("culprit", "unknown") level = error_context.get("level", "error") + route_labels = { + key: value + for key in ( + "awoooi_product_id", + "awoooi_signal_family", + "awoooi_route_severity", + ) + if ( + value := _sentry_event_tag( + {"tags": error_context.get("tags", [])}, + key, + ) + ) + } + telegram_route_context = get_trusted_alert_canonical_route_context( + route_labels, + source="sentry_webhook", + source_severity=str(level), + ) # 發送 Sentry 告警卡片 (含 Y/n 按鈕) # TODO(2026-04-05): Sentry 路徑無 incident_id，待 Sentry→Incident 關聯後補傳 @@ -663,7 +685,10 @@ async def send_sentry_telegram_alert( anomaly_frequency=anomaly_frequency, # 2026-04-02 ogt: 修復 ai_provider 未傳遞 → Telegram 顯示「AI 仲裁判定」而非具體模型名稱 ai_provider=analysis.analyzed_by if analysis else "", - notification_type="TYPE-3", + notification_type="UNKNOWN", + product_id=(telegram_route_context or {}).get("product_id"), + signal_family=(telegram_route_context or {}).get("signal_family"), + route_severity=(telegram_route_context or {}).get("severity"), ) if _telegram_send_delivery_succeeded(delivery_result): diff --git a/apps/api/src/api/v1/signoz_webhook.py b/apps/api/src/api/v1/signoz_webhook.py index cb8394444..21d6cee4c 100644 --- a/apps/api/src/api/v1/signoz_webhook.py +++ b/apps/api/src/api/v1/signoz_webhook.py @@ -552,6 +552,10 @@ async def send_signoz_telegram( service_name = labels.get("service_name", labels.get("service", "unknown")) summary = annotations.get("summary", f"SignOz Alert: {alert_name}") description = annotations.get("description", "") + # This public SignOz receiver does not yet authenticate the monitoring + # source. It may persist/normalize the signal, but Telegram egress is + # fail-closed until an authenticated ingress supplies route identity. + telegram_route_context: dict[str, str] | None = None delivery_result = await telegram.send_approval_card( approval_id=approval_id, @@ -572,7 +576,10 @@ async def send_signoz_telegram( # 2026-04-02 ogt: 修復 ai_provider 未傳遞 → Telegram 顯示「AI 仲裁判定」而非具體模型名稱 ai_provider=ai_provider if ai_provider != "none" else "", incident_id=incident_id, - notification_type="TYPE-3", + notification_type="UNKNOWN", + product_id=(telegram_route_context or {}).get("product_id"), + signal_family=(telegram_route_context or {}).get("signal_family"), + route_severity=(telegram_route_context or {}).get("severity"), ) if _telegram_send_delivery_succeeded(delivery_result): diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 0c8bb69e8..3089895fe 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -90,6 +90,10 @@ from src.services.incident_service import ( get_incident_service, ) +from src.services.notification_matrix import ( + get_trusted_alert_canonical_route_context, +) + # Phase 5: OpenClaw AI Engine from src.services.openclaw import get_openclaw from src.services.playbook_match_resolver import resolve_playbook_id_for_alert @@ -994,6 +998,7 @@ async def _push_to_telegram_background( repair_candidate_promotion_summary: str = "", repair_candidate_work_item_href: str = "", repair_candidate_work_item_id: str = "", + telegram_route_context: dict[str, str] | None = None, ) -> None: """ 背景任務: 推送待簽核卡片到 Telegram (v7.0 含 SignOz 整合) @@ -1116,6 +1121,9 @@ async def _push_to_telegram_background( repair_candidate_promotion_summary=repair_candidate_promotion_summary, repair_candidate_work_item_href=repair_candidate_work_item_href, repair_candidate_work_item_id=repair_candidate_work_item_id, + product_id=(telegram_route_context or {}).get("product_id"), + signal_family=(telegram_route_context or {}).get("signal_family"), + route_severity=(telegram_route_context or {}).get("severity"), ) if not _telegram_send_delivery_succeeded(delivery_result): @@ -1934,6 +1942,11 @@ async def receive_alert( elif isinstance(first_suggestion, dict): auto_tuning_cmd = first_suggestion.get("kubectl_or_config", "") + telegram_route_context = get_trusted_alert_canonical_route_context( + alert.labels, + source="hmac_alert_webhook", + source_severity=alert.severity, + ) background_tasks.add_task( _push_to_telegram_background, approval_id=str(approval.id), @@ -1961,6 +1974,8 @@ async def receive_alert( ai_provider=ai_provider, # 2026-04-08 ogt: 補傳 incident_id 以啟用詳情/重診/歷史按鈕 incident_id=_cs1_incident_id, + notification_type="UNKNOWN", + telegram_route_context=telegram_route_context, # /alerts 路徑沒有 notification_type（非 Alertmanager 路徑），不需 TYPE-4D routing # ADR-075: alert_type → category 對應，啟用動態按鈕 alert_category={ @@ -2074,6 +2089,28 @@ def is_internal_ip(client_ip: str) -> bool: return False +def _trusted_internal_alertmanager_source(request: Request) -> tuple[bool, str]: + """Accept Alertmanager only when the peer and full proxy chain are private.""" + + peer_ip = request.client.host if request.client else "unknown" + if not is_internal_ip(peer_ip): + return False, peer_ip + + forwarded_chain = [ + value.strip() + for value in request.headers.get("X-Forwarded-For", "").split(",") + if value.strip() + ] + if not forwarded_chain: + return True, peer_ip + if not all(is_internal_ip(value) for value in forwarded_chain): + first_external = next( + value for value in forwarded_chain if not is_internal_ip(value) + ) + return False, first_external + return True, forwarded_chain[0] + + async def _process_new_alert_background( alert_context: dict, alert_id: str, @@ -2104,6 +2141,11 @@ async def _process_new_alert_background( "fingerprint": fingerprint, "alert_id": alert_id, } + telegram_route_context = get_trusted_alert_canonical_route_context( + alert_labels, + source="prometheus_alertmanager", + source_severity=severity, + ) rule_response = match_rule(alert_context) should_bypass_llm = _should_use_alertmanager_rule_first(rule_response, alert_category) @@ -2349,6 +2391,7 @@ async def _process_new_alert_background( notification_type=notification_type, alert_category=alert_category, fingerprint=fingerprint, + telegram_route_context=telegram_route_context, ) record_alert_chain_success("alertmanager") @@ -2599,6 +2642,7 @@ async def _process_new_alert_background( notification_type=notification_type, alert_category=alert_category, fingerprint=fingerprint, + telegram_route_context=telegram_route_context, ) record_alert_chain_success("alertmanager") @@ -2988,6 +3032,7 @@ async def _process_new_alert_background( if isinstance(_approval_metadata_cs4.get("repair_candidate_draft_package"), dict) else "" ), + telegram_route_context=telegram_route_context, ) except Exception as e: @@ -3042,12 +3087,10 @@ async def alertmanager_webhook( ) # 取得客戶端 IP - client_ip = request.client.host if request.client else "unknown" - forwarded_for = request.headers.get("X-Forwarded-For", "").split(",")[0].strip() - actual_ip = forwarded_for or client_ip + internal_source, actual_ip = _trusted_internal_alertmanager_source(request) # 內網檢查 - if not is_internal_ip(actual_ip): + if not internal_source: logger.warning( "alertmanager_external_rejected", client_ip=actual_ip, diff --git a/apps/api/src/services/notification_matrix.py b/apps/api/src/services/notification_matrix.py index 921e23db2..b6ca1455b 100644 --- a/apps/api/src/services/notification_matrix.py +++ b/apps/api/src/services/notification_matrix.py @@ -11,6 +11,7 @@ from __future__ import annotations import json import re +from collections.abc import Mapping from dataclasses import dataclass from enum import Enum from pathlib import Path @@ -89,6 +90,23 @@ _SHARED_SRE_SIGNAL_FAMILIES = frozenset( "incident_lifecycle", } ) +_TRUSTED_ALERT_ROUTE_SOURCES = frozenset( + { + "hmac_alert_webhook", + "prometheus_alertmanager", + "sentry_webhook", + } +) +_ROUTE_SEVERITY_BY_SOURCE_VALUE = { + "critical": "P0", + "fatal": "P0", + "error": "P1", + "high": "P1", + "warning": "P2", + "medium": "P2", + "info": "P3", + "low": "P3", +} _REQUIRED_ROUTE_FIELDS = frozenset( { "route_id", @@ -230,6 +248,44 @@ def get_legacy_canonical_route_context( } +def get_trusted_alert_canonical_route_context( + labels: Mapping[str, object] | None, + *, + source: str, + source_severity: str, +) -> dict[str, str] | None: + """Extract an explicit product route from controlled, namespaced labels. + + Product monitoring must never inherit the legacy TYPE-3 AWOOOI route. A + recognized ingress may opt into canonical routing only with the committed + ``awoooi_*`` label triplet. Missing or malformed labels return ``None`` so + the caller can emit a durable blocked/no-egress receipt. + """ + + if source not in _TRUSTED_ALERT_ROUTE_SOURCES or not isinstance(labels, Mapping): + return None + + product_id = str(labels.get("awoooi_product_id") or "").strip() + signal_family = str(labels.get("awoooi_signal_family") or "").strip() + if not product_id or not signal_family: + return None + + explicit_severity = str(labels.get("awoooi_route_severity") or "").strip().upper() + route_severity = explicit_severity or _ROUTE_SEVERITY_BY_SOURCE_VALUE.get( + str(source_severity or "").strip().lower(), + "", + ) + if route_severity not in _ALLOWED_SEVERITIES: + return None + + return { + "product_id": product_id, + "signal_family": signal_family, + "severity": route_severity, + "route_source": source, + } + + def resolve_chat_ids( notification_type: str, dm_chat_id: str, diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 25bc95342..4d3815a79 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -7163,6 +7163,9 @@ class TelegramGateway: repair_candidate_promotion_summary: str = "", repair_candidate_work_item_href: str = "", repair_candidate_work_item_id: str = "", + product_id: str | None = None, + signal_family: str | None = None, + route_severity: str | None = None, ) -> dict: """ 推送待簽核卡片到 Telegram (v7.0 含 SignOz 整合) @@ -7184,6 +7187,8 @@ class TelegramGateway: signoz_trace_url: 動態時間參數的 Trace URL auto_tuning_command: kubectl 調優指令 anomaly_frequency: 異常頻率統計 (ADR-037) + product_id/signal_family/route_severity: namespaced canonical + product route. When present it supersedes legacy TYPE-*. Returns: dict: Telegram API 回應 @@ -7306,13 +7311,20 @@ class TelegramGateway: # Unknown/missing notification_type is intentionally no-egress. The # central sender resolves only allowlisted legacy types via the # canonical registry and binds the actual runtime destination there. - payload = { + payload: dict[str, object] = { "text": text, "parse_mode": "HTML", "reply_markup": keyboard, "disable_web_page_preview": True, # 避免 SignOz URL 預覽 - _LEGACY_NOTIFICATION_TYPE_KEY: notification_type or "UNKNOWN", } + if any((product_id, signal_family, route_severity)): + payload[_CANONICAL_ROUTE_CONTEXT_KEY] = { + "product_id": product_id or "", + "signal_family": signal_family or "", + "severity": route_severity or "", + } + else: + payload[_LEGACY_NOTIFICATION_TYPE_KEY] = notification_type or "UNKNOWN" logger.info( "telegram_approval_card_sending", diff --git a/apps/api/tests/test_alertmanager_webhook_metrics.py b/apps/api/tests/test_alertmanager_webhook_metrics.py index 3da038a96..f9817361d 100644 --- a/apps/api/tests/test_alertmanager_webhook_metrics.py +++ b/apps/api/tests/test_alertmanager_webhook_metrics.py @@ -75,3 +75,38 @@ async def test_alertmanager_external_reject_records_error_metric() -> None: 'awoooi_webhook_requests_total{source="alertmanager",status="error"}' in _metrics_text() ) + + +@pytest.mark.asyncio +async def test_alertmanager_rejects_external_forwarded_source_behind_private_peer() -> None: + before = _webhook_count("error") + + with pytest.raises(HTTPException) as exc_info: + await alertmanager_webhook( + _request( + client_host="10.42.0.10", + headers=[(b"x-forwarded-for", b"8.8.8.8, 10.42.0.1")], + ), + AlertmanagerPayload(status="resolved", alerts=[]), + BackgroundTasks(), + ) + + assert exc_info.value.status_code == 403 + assert _webhook_count("error") == before + 1 + + +@pytest.mark.asyncio +async def test_alertmanager_accepts_private_forwarded_chain_from_private_peer() -> None: + before = _webhook_count("success") + + response = await alertmanager_webhook( + _request( + client_host="10.42.0.10", + headers=[(b"x-forwarded-for", b"192.168.0.110, 10.42.0.1")], + ), + AlertmanagerPayload(status="resolved", alerts=[]), + BackgroundTasks(), + ) + + assert response.success is True + assert _webhook_count("success") == before + 1 diff --git a/apps/api/tests/test_telegram_canonical_routing_registry.py b/apps/api/tests/test_telegram_canonical_routing_registry.py index 75f7e36c8..88649644b 100644 --- a/apps/api/tests/test_telegram_canonical_routing_registry.py +++ b/apps/api/tests/test_telegram_canonical_routing_registry.py @@ -2,11 +2,14 @@ from __future__ import annotations import copy import json +import re from pathlib import Path import pytest +import yaml from src.services.notification_matrix import ( + get_trusted_alert_canonical_route_context, load_canonical_telegram_routing_registry, resolve_canonical_telegram_route, ) @@ -24,6 +27,158 @@ EXPECTED_PRODUCTS = { "vibework", "vtuber", } +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def test_namespaced_product_ingress_route_is_explicit_and_policy_blocked() -> None: + context = get_trusted_alert_canonical_route_context( + { + "awoooi_product_id": "momo-pro", + "awoooi_signal_family": "crawler_and_data_freshness", + "awoooi_route_severity": "P2", + }, + source="prometheus_alertmanager", + source_severity="warning", + ) + + assert context == { + "product_id": "momo-pro", + "signal_family": "crawler_and_data_freshness", + "severity": "P2", + "route_source": "prometheus_alertmanager", + } + assert resolve_canonical_telegram_route(**{ + key: context[key] for key in ("product_id", "signal_family", "severity") + })["decision"] == "block" + + +@pytest.mark.parametrize( + ("labels", "source", "severity"), + [ + ({}, "prometheus_alertmanager", "critical"), + ( + { + "awoooi_product_id": "momo-pro", + "awoooi_signal_family": "crawler_and_data_freshness", + }, + "untrusted_ingress", + "critical", + ), + ( + { + "awoooi_product_id": "awoooi", + "awoooi_signal_family": "incident_lifecycle", + "awoooi_route_severity": "P1", + }, + "signoz_webhook", + "critical", + ), + ( + { + "awoooi_product_id": "momo-pro", + "awoooi_signal_family": "crawler_and_data_freshness", + "awoooi_route_severity": "SEV-1", + }, + "prometheus_alertmanager", + "critical", + ), + ], +) +def test_product_ingress_route_fails_closed_without_complete_trusted_context( + labels: dict[str, str], source: str, severity: str +) -> None: + assert get_trusted_alert_canonical_route_context( + labels, + source=source, + source_severity=severity, + ) is None + + +@pytest.mark.parametrize("rule_file", ["alerts-unified.yml", "alerts.yml"]) +def test_product_alert_rules_carry_canonical_route_labels(rule_file: str) -> None: + payload = yaml.safe_load((REPO_ROOT / "ops/monitoring" / rule_file).read_text()) + rules = { + rule["alert"]: rule["labels"] + for group in payload["groups"] + for rule in group.get("rules", []) + if "alert" in rule + } + expected = { + "OpenClawDown": ("clawbot-openclaw", "raw_monitoring_alert", "P1"), + "MoWoooWorkDown": ("momo-pro", "scheduler_and_sales_pipeline", "P1"), + "TsenyangWebsiteDown": ( + "tsenyang-website", + "public_site_and_agent_action_health", + "P1", + ), + "StockWoooWorkDown": ( + "stockplatform-v2", + "market_data_and_recommendation_freshness", + "P1", + ), + "BitanWoooWorkDown": ( + "bitan-pharmacy", + "container_and_product_health", + "P1", + ), + "MomoScraperSuccessLow": ( + "momo-pro", + "crawler_and_data_freshness", + "P2", + ), + } + for alertname, route in expected.items(): + labels = rules[alertname] + assert ( + labels["awoooi_product_id"], + labels["awoooi_signal_family"], + labels["awoooi_route_severity"], + ) == route + + +def test_specialized_openclaw_rule_cannot_fall_back_to_shared_sre() -> None: + payload = yaml.safe_load( + (REPO_ROOT / "k8s/monitoring/k3s-alerts-supplemental.yaml").read_text() + ) + rule = next( + rule + for group in payload["groups"] + for rule in group.get("rules", []) + if rule.get("alert") == "OpenClawDown" + ) + labels = rule["labels"] + assert { + key: labels[key] + for key in ( + "awoooi_product_id", + "awoooi_signal_family", + "awoooi_route_severity", + ) + } == { + "awoooi_product_id": "clawbot-openclaw", + "awoooi_signal_family": "raw_monitoring_alert", + "awoooi_route_severity": "P2", + } + + +def test_active_source_has_no_raw_numeric_telegram_destination_defaults() -> None: + candidate_paths = [ + REPO_ROOT / "docker-compose.yml", + REPO_ROOT / "k8s/awoooi-prod/04-configmap.yaml", + REPO_ROOT / "scripts/ops/deploy-docker-health-monitor.sh", + REPO_ROOT / "scripts/ops/docker-health-monitor.sh", + *(REPO_ROOT / ".gitea/workflows").glob("*.y*ml"), + ] + raw_numeric_identifier = re.compile(r"-?\d{8,}") + violations: list[str] = [] + for path in candidate_paths: + for line_number, line in enumerate(path.read_text().splitlines(), 1): + if ( + any(key in line for key in ("CHAT_ID", "USER_WHITELIST")) + and raw_numeric_identifier.search(line) + ): + violations.append(f"{path.relative_to(REPO_ROOT)}:{line_number}") + assert violations == [] def test_registry_covers_all_11_products_and_reconciles_rollups() -> None: diff --git a/apps/api/tests/test_telegram_canonical_sender_gate.py b/apps/api/tests/test_telegram_canonical_sender_gate.py index 02cc352f6..42658e58b 100644 --- a/apps/api/tests/test_telegram_canonical_sender_gate.py +++ b/apps/api/tests/test_telegram_canonical_sender_gate.py @@ -847,3 +847,30 @@ async def test_allowed_type3_approval_persists_runtime_chat_without_public_id( receipt = result["_awoooi_canonical_route_receipt"] assert receipt["destination_alias"] == "awoooi_sre_war_room" assert "chat_id" not in receipt + + +@pytest.mark.asyncio +async def test_explicit_blocked_product_route_supersedes_legacy_type3( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + _patch_approval_dependencies(monkeypatch, gateway) + + result = await gateway.send_approval_card( + approval_id="APR-PRODUCT", + risk_level="medium", + resource_name="momo-crawler", + root_cause="freshness degraded", + suggested_action="inspect evidence", + notification_type="TYPE-3", + product_id="momo-pro", + signal_family="crawler_and_data_freshness", + route_severity="P2", + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awooop_provider_send_performed"] is False + assert client.posts == [] + receipt = result["_awoooi_canonical_route_receipt"] + assert receipt["product_id"] == "momo-pro" + assert receipt["signal_family"] == "crawler_and_data_freshness" diff --git a/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py index b92866146..ad6bfe267 100644 --- a/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py +++ b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py @@ -494,7 +494,7 @@ async def test_background_no_send_never_confirms_or_writes_sent( @pytest.mark.asyncio @pytest.mark.parametrize("module_name", ["sentry", "signoz"]) -async def test_monitoring_webhooks_use_type3_and_do_not_false_log_sent( +async def test_unclassified_monitoring_webhooks_fail_closed_and_do_not_false_log_sent( monkeypatch: pytest.MonkeyPatch, module_name: str, ) -> None: @@ -533,7 +533,10 @@ async def test_monitoring_webhooks_use_type3_and_do_not_false_log_sent( sent_event = "signoz_telegram_sent" no_send_event = "signoz_telegram_no_send" - assert captured[0]["notification_type"] == "TYPE-3" + assert captured[0]["notification_type"] == "UNKNOWN" + assert captured[0]["product_id"] is None + assert captured[0]["signal_family"] is None + assert captured[0]["route_severity"] is None assert ("warning", no_send_event) in logger.events assert ("info", sent_event) not in logger.events diff --git a/docker-compose.yml b/docker-compose.yml index b3b9aaf62..932261627 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,8 +76,8 @@ services: CORS_ORIGINS: '["http://localhost:3000","http://localhost:3001","http://localhost:3002","http://localhost:3003","http://web:3000"]' # Telegram Gateway (Phase 5.5) OPENCLAW_TG_BOT_TOKEN: "${OPENCLAW_TG_BOT_TOKEN:-}" - OPENCLAW_TG_CHAT_ID: "5619078117" - OPENCLAW_TG_USER_WHITELIST: "5619078117" + OPENCLAW_TG_CHAT_ID: "${OPENCLAW_TG_CHAT_ID:-}" + OPENCLAW_TG_USER_WHITELIST: "${OPENCLAW_TG_USER_WHITELIST:-}" # External Services (使用 host.docker.internal 存取宿主機服務) OLLAMA_URL: http://host.docker.internal:11434 OPENCLAW_URL: http://host.docker.internal:8088 diff --git a/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md index 53b3a781b..22200319a 100644 --- a/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md +++ b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md @@ -12,6 +12,40 @@ - 未知 product/signal/severity/destination 一律 no-egress；不再沿用 ADR-093 的 unknown-to-shared-group fallback。 - `AwoooI SRE 戰情室` 只收 shared infrastructure、security、DR 與 P0/P1 incident lifecycle。raw P2/P3、marketing、成功 heartbeat、未驗證 recommendation 和 bot 對話均不得進入。 +## 監控與告警來源總帳 + +| source truth | 數量 | 角色與目前判定 | +|---|---:|---| +| `ops/monitoring/alerts-unified.yml` | 23 groups / 145 alert declarations | production primary Prometheus rule source；產品訊號必須帶 namespaced canonical route labels | +| `ops/monitoring/alerts.yml` | 81 declarations | legacy duplicate source，不加入 production primary 總數；保留相同產品 labels 防止誤復用 | +| 11 個 specialized Prometheus / SignOz rule files | 112 declarations | K3s、DB、flywheel、MinIO/Kali、NVIDIA、Ollama、SLO、agent latency、SignOz 等專項訊號 | +| primary + specialized 合計 | 257 declarations / 226 unique alert names | 29 個 alert name 在兩個以上來源重複，extra declarations 31；重複不是多送授權，仍由 grouping/fingerprint/route policy 收斂 | +| AWOOOI API `alert_rules` | 29 matching/action rules | 用於分類與修復候選，不是新的 signal generator，不重複計入 257 | +| Telegram egress scanner | 805 files / 50 normalized gateway callsites | active direct Bot API 0；7 個 direct calls 全在 GitHub-frozen legacy source，禁止執行 | + +目前可明確歸戶的產品告警如下；未列出的 shared infrastructure/security/DR/K3s/DB/host 規則屬 `awoooi`，但只有 P0/P1 allowlisted lifecycle 能進 SRE 群： + +| product_id | 明確 signal / alert | canonical signal family | 現行 egress | +|---|---|---|---| +| `2026fifa` | repository 中沒有產品專屬 Prometheus rule | `data_freshness_and_recommendation` | blocked | +| `agent-bounty-protocol` | registry 有 A2A、traffic security、treasury/onboarding 3 lanes；沒有專屬 Prometheus rule | 對應 3 個 registry families | blocked，缺 durable receipt | +| `awooogo` | 無產品專屬 rule | `*` | disabled | +| `awoooi` | 145 primary + 112 specialized declarations 的 shared/core owner | shared infrastructure / security / DR / incident lifecycle | 僅 P0/P1 allowlisted route allowed | +| `bitan-pharmacy` | `BitanWoooWorkDown` | `container_and_product_health` | blocked | +| `clawbot-openclaw` | `OpenClawDown` | `raw_monitoring_alert` | blocked；不得借用 AWOOOI TYPE-3 | +| `momo-pro` | `MoWoooWorkDown`、`MomoScraperSuccessLow` | scheduler/sales、crawler/data freshness | blocked；configured destination 仍 `chat_not_found` | +| `stockplatform-v2` | `StockWoooWorkDown` | `market_data_and_recommendation_freshness` | blocked | +| `tsenyang-website` | `TsenyangWebsiteDown` | `public_site_and_agent_action_health` | blocked | +| `vibework` | 無專屬 rule；`VibeAIAgent` 不等於 standalone VibeWork | `*` | not implemented | +| `vtuber` | 無產品專屬 rule | `*` | disabled | + +## Source-side route corrections + +- Alertmanager 產品規則現在帶 `awoooi_product_id`、`awoooi_signal_family`、`awoooi_route_severity`；gateway 以這組明確 context 覆蓋 legacy `TYPE-3`，所以 blocked product 不會再落到 shared SRE。 +- SignOz public receiver 尚無 source authentication，因此即使 payload 自稱 canonical labels 也固定 no-egress；Sentry（signature verified）與 HMAC generic webhook 缺完整 namespaced context 時一律 `UNKNOWN / blocked_no_egress`。Alertmanager 另要求 direct peer 與完整 forwarded chain 都是 private IP，不再信任單一可偽造的 forwarded value。 +- `docker-compose`、production ConfigMap、docker-health scripts 與 7 個 Gitea workflows 的 11 個 raw destination defaults 已移除；runtime 只可由既有 secret/env alias 注入，缺值即 fail closed。白名單也不再內嵌 numeric default。 +- 本次只改 source policy，沒有讀 token、呼叫 Bot API、改 Telegram 群組、送測試訊息或偽造 delivery receipt。 + ## 11 產品路由清冊 | 產品 | 目前 proven route | 建議 canonical route | blocked unknown / egress status | source risk | receipt strength | diff --git a/k8s/awoooi-prod/04-configmap.yaml b/k8s/awoooi-prod/04-configmap.yaml index e18036039..20a940d70 100644 --- a/k8s/awoooi-prod/04-configmap.yaml +++ b/k8s/awoooi-prod/04-configmap.yaml @@ -107,7 +107,6 @@ data: # 2026-04-03 ogt: SRE 戰情室群組三頭政治 (Triumvirate ADR-053) # OPENCLAW_BOT_TOKEN / NEMOTRON_BOT_TOKEN 在 Secrets 中配置 # ============================================================================ - SRE_GROUP_CHAT_ID: "-1003711974679" # 2026-07-10 Codex: Agent99 direct SRE relay. Alertmanager still lands in # AWOOI API first; the sanitized bridge then POSTs to 99 so Agent99 can queue # allowlisted recovery and write evidence/KM. Optional relay token stays in diff --git a/k8s/monitoring/k3s-alerts-supplemental.yaml b/k8s/monitoring/k3s-alerts-supplemental.yaml index b621c6194..8c860e39b 100644 --- a/k8s/monitoring/k3s-alerts-supplemental.yaml +++ b/k8s/monitoring/k3s-alerts-supplemental.yaml @@ -39,6 +39,9 @@ groups: for: 2m labels: severity: warning + awoooi_product_id: clawbot-openclaw + awoooi_signal_family: raw_monitoring_alert + awoooi_route_severity: P2 team: ops component: openclaw annotations: diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 22b804a13..20b4b368c 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -703,6 +703,9 @@ groups: for: 2m labels: severity: critical + awoooi_product_id: clawbot-openclaw + awoooi_signal_family: raw_monitoring_alert + awoooi_route_severity: P1 layer: docker-188 component: openclaw host: "188" @@ -1133,6 +1136,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: momo-pro + awoooi_signal_family: scheduler_and_sales_pipeline + awoooi_route_severity: P1 layer: external component: momo-pro-system host: "188" @@ -1148,6 +1154,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: tsenyang-website + awoooi_signal_family: public_site_and_agent_action_health + awoooi_route_severity: P1 layer: external component: tsenyang-website host: "188" @@ -1162,6 +1171,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: stockplatform-v2 + awoooi_signal_family: market_data_and_recommendation_freshness + awoooi_route_severity: P1 layer: external component: stock-platform host: "110" @@ -1176,6 +1188,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: bitan-pharmacy + awoooi_signal_family: container_and_product_health + awoooi_route_severity: P1 layer: external component: bitan-app host: "188" @@ -1289,6 +1304,9 @@ groups: for: 10m labels: severity: warning + awoooi_product_id: momo-pro + awoooi_signal_family: crawler_and_data_freshness + awoooi_route_severity: P2 layer: docker-110 auto_repair: "false" alert_category: business diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index 9dccc313f..3e21b64f2 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -387,6 +387,9 @@ groups: for: 2m labels: severity: critical + awoooi_product_id: clawbot-openclaw + awoooi_signal_family: raw_monitoring_alert + awoooi_route_severity: P1 layer: docker-188 component: openclaw host: "188" @@ -817,6 +820,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: momo-pro + awoooi_signal_family: scheduler_and_sales_pipeline + awoooi_route_severity: P1 layer: external component: momo-pro-system host: "188" @@ -832,6 +838,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: tsenyang-website + awoooi_signal_family: public_site_and_agent_action_health + awoooi_route_severity: P1 layer: external component: tsenyang-website host: "188" @@ -846,6 +855,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: stockplatform-v2 + awoooi_signal_family: market_data_and_recommendation_freshness + awoooi_route_severity: P1 layer: external component: stock-platform host: "110" @@ -860,6 +872,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: bitan-pharmacy + awoooi_signal_family: container_and_product_health + awoooi_route_severity: P1 layer: external component: bitan-app host: "188" @@ -937,6 +952,9 @@ groups: for: 10m labels: severity: warning + awoooi_product_id: momo-pro + awoooi_signal_family: crawler_and_data_freshness + awoooi_route_severity: P2 layer: docker-110 auto_repair: "false" alert_category: business diff --git a/scripts/ops/deploy-docker-health-monitor.sh b/scripts/ops/deploy-docker-health-monitor.sh index 76c7df04e..e3c2fe902 100755 --- a/scripts/ops/deploy-docker-health-monitor.sh +++ b/scripts/ops/deploy-docker-health-monitor.sh @@ -86,7 +86,7 @@ deploy_to_host() { AWOOOI_API_URL=https://awoooi.wooo.work TELEGRAM_BOT_TOKEN=CHANGE_ME -SRE_GROUP_CHAT_ID=-1003711974679 +SRE_GROUP_CHAT_ID=CHANGE_ME SEND_COOLDOWN_SECONDS=300 ACTION_COOLDOWN_SECONDS=300 NOTIFY_COOLDOWN_SECONDS=1800 diff --git a/scripts/ops/docker-health-monitor.sh b/scripts/ops/docker-health-monitor.sh index db489d289..040712a9e 100755 --- a/scripts/ops/docker-health-monitor.sh +++ b/scripts/ops/docker-health-monitor.sh @@ -23,7 +23,7 @@ if [[ -f "$SECRETS_FILE" ]]; then fi : "${AWOOOI_API_URL:=https://awoooi.wooo.work}" -: "${SRE_GROUP_CHAT_ID:=-1003711974679}" +: "${SRE_GROUP_CHAT_ID:=}" : "${LOG_FILE:=/var/log/docker-health-monitor.log}" : "${SEND_COOLDOWN_SECONDS:=300}" : "${ACTION_COOLDOWN_SECONDS:=${SEND_COOLDOWN_SECONDS}}"