diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 2f2eddea0..fad8a14f4 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -5,17 +5,18 @@ - 以 `P0-OBS-001-20260714T191800+0800` 進行受控修復:`.110` 只重建 stateless `signoz-otel-collector`,改用已驗證的 static config;SigNoz UI、ClickHouse、Zookeeper 的 container ID 與啟動時間未變。 - 新增 `.188:24318 -> .110:4318` 暫時相容橋接,僅允許 `.120/.121` 來源,具備 check / apply / status / rollback 與 `2026-08-15` 到期日。 - K3s OTEL DaemonSet 新增 `file_storage`、filelog offset 持久化、exporter WAL、`queue_size=10000`、無限時 retry、60 秒 termination grace 與逐節點 rolling update。 -- Event Exporter 新增最小化 metadata-only RBAC,僅允許 `get` 事件引用的 Pod、Deployment、ReplicaSet、DaemonSet、Job、CronJob 與 Argo Application;明確不授權 Secret / ConfigMap 或任何寫入動作。 +- Event Exporter 改為 `omitLookup=true`,撤回不必要的 workload / Argo metadata RBAC,只保留 Events / Namespaces watch;同時將 `maxEventAgeSeconds` 由預設 5 秒提高為 120 秒,避免 deploy burst 與 client throttling 丟棄 RCA 事件。 - 修正 Event Exporter 的 durable write 缺口:舊平面 webhook JSON 雖收到 OTLP HTTP `200`,實際未產生 log;現改為標準 OTLP JSON envelope 與單一 receiver,避免未列出的 Normal 事件重複送出。 - 將現行 `0.96.0` image 由 K3s cache 直接匯入 Harbor,未對外下載;production 已 pin 至 `192.168.0.110:5000/awoooi/otel-collector-contrib@sha256:8edf386c...`。 - 新增獨立 ingestion verifier,以同一 run 綁定 K3s queue、`.188` bridge、`.110` listener 與 ClickHouse exact-marker 讀回。 **已跑驗證**: -- 精準 pytest:`11 passed`;三支 shell 的 `bash -n` 與 `git diff --check` 通過。 +- 精準 pytest:`12 passed`;三支 shell 的 `bash -n` 與 `git diff --check` 通過。 - `kubectl apply --dry-run=server`、exact live digest `file_storage` canary、Harbor digest pull canary 全數通過;canary 顯示 `Extension started: file_storage`、`Initializing new persistent queue`、`Everything is ready`。 - DaemonSet generation `4 -> 5` 完成逐節點 rollout;兩個 collector 均 `Ready`、`restarts=0`、queue capacity `10000`、queue size `0`。 - Synthetic OTLP 從 `.120 -> .188:24318 -> .110:4318 -> signoz_logs.logs_v2` 回讀 exact count `1`;verifier terminal 為 `queue_zero_failure_deltas_zero_window_15s`,兩個 production collector 後續連續超過 11 分鐘 exporter error `0`、queue `0`、send-failed `0`。 - `P0-OBS-001-20260714T201500+0800-event-otlp-layout` synthetic Kubernetes Warning Event 在 ClickHouse 回讀 exact body `1`、結構化 reason / service attributes `1`;新 Pod `Ready=true`、`restarts=0`、metadata forbidden `0`、webhook delivery error `0`。 +- Live deploy burst 曾留下多筆 `event age=5-11s ... discarded`,並使 quota `FailedCreate` 未落盤;受控修復 `P0-OBS-001-20260714T204700+0800-event-burst-loss-retry2` 以 30 秒 old Event 驗證 ClickHouse exact body `1`、attributes `1`、discard `0`、lookup forbidden `0`。第一次 verifier 因 invalid Event schema 在建立前失敗,已自動回滾並讀回舊 ConfigMap / RBAC / Ready Pod 後才重試。 **仍維持**: - `setupCompleted=false` 與 SigNoz organization 未初始化仍是治理缺口;static override 與 `.188` 橋接是可回滾、有到期日的暫時控制,不是 Foundry 遷移完成。 diff --git a/k8s/observability/event-exporter.yaml b/k8s/observability/event-exporter.yaml index 0b62f5c78..f4227ed38 100644 --- a/k8s/observability/event-exporter.yaml +++ b/k8s/observability/event-exporter.yaml @@ -24,20 +24,6 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "watch", "list"] - # P0-OBS-001: event-exporter enriches events by reading the referenced object. - # Keep this metadata-only: no list/watch here and never grant Secret access. - - apiGroups: [""] - resources: ["pods", "nodes", "services", "persistentvolumeclaims", "persistentvolumes"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] - verbs: ["get"] - - apiGroups: ["batch"] - resources: ["jobs", "cronjobs"] - verbs: ["get"] - - apiGroups: ["argoproj.io"] - resources: ["applications"] - verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -62,6 +48,11 @@ data: # Phase O-2.2: K8s Event → SigNoz (OTLP HTTP) logLevel: info logFormat: json + # P0-OBS-001: deploy bursts can reach the watcher 5-15 seconds late. + # Preserve them for RCA and avoid one API GET per involved object; the OTLP + # layout below only consumes fields already present on the Event itself. + maxEventAgeSeconds: 120 + omitLookup: true route: routes: # Warning/Error 全量保留;只排除已知高頻、低診斷價值的 Normal 事件。 diff --git a/scripts/reboot-recovery/tests/test_event_exporter_metadata_rbac.py b/scripts/reboot-recovery/tests/test_event_exporter_metadata_rbac.py index 5fcfcf5c0..4ecc6eb5b 100644 --- a/scripts/reboot-recovery/tests/test_event_exporter_metadata_rbac.py +++ b/scripts/reboot-recovery/tests/test_event_exporter_metadata_rbac.py @@ -9,31 +9,24 @@ ROOT = Path(__file__).resolve().parents[3] MANIFEST = ROOT / "k8s" / "observability" / "event-exporter.yaml" -def test_event_exporter_has_minimal_metadata_enrichment_reads() -> None: +def test_event_exporter_watches_only_events_and_namespaces() -> None: documents = [ doc for doc in yaml.safe_load_all(MANIFEST.read_text(encoding="utf-8")) if doc ] role = next(doc for doc in documents if doc["kind"] == "ClusterRole") rules = role["rules"] - - expected = { - ("", "pods"), - ("apps", "deployments"), - ("apps", "replicasets"), - ("apps", "daemonsets"), - ("batch", "jobs"), - ("batch", "cronjobs"), - ("argoproj.io", "applications"), - } - granted = { - (api_group, resource) - for rule in rules - for api_group in rule["apiGroups"] - for resource in rule["resources"] - if rule["verbs"] == ["get"] - } - assert expected <= granted - assert all(resource not in {"secrets", "configmaps"} for _, resource in granted) + assert rules == [ + { + "apiGroups": [""], + "resources": ["events"], + "verbs": ["get", "watch", "list"], + }, + { + "apiGroups": [""], + "resources": ["namespaces"], + "verbs": ["get", "watch", "list"], + }, + ] def test_event_exporter_metadata_rules_never_write_or_delete() -> None: @@ -89,3 +82,9 @@ def test_event_exporter_has_one_route_and_drops_only_known_normal_noise() -> Non "Created", "Started", } + + +def test_event_exporter_keeps_late_events_without_object_lookups() -> None: + config = _event_exporter_config() + assert config["omitLookup"] is True + assert config["maxEventAgeSeconds"] == 120