From fefe810df46615c84223621b9f746c77beea2d74 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 20:16:42 +0800 Subject: [PATCH] fix(observability): persist Kubernetes events through OTLP --- docs/LOGBOOK.md | 5 +- k8s/observability/event-exporter.yaml | 106 ++++++++++++------ .../test_event_exporter_metadata_rbac.py | 91 +++++++++++++++ 3 files changed, 165 insertions(+), 37 deletions(-) create mode 100644 scripts/reboot-recovery/tests/test_event_exporter_metadata_rbac.py diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index fc15d45e3..2f2eddea0 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -5,14 +5,17 @@ - 以 `P0-OBS-001-20260714T191800+0800` 進行受控修復：`.110` 只重建 stateless `signoz-otel-collector`，改用已驗證的 static config；SigNoz UI、ClickHouse、Zookeeper 的 container ID 與啟動時間未變。 - 新增 `.188:24318 -> .110:4318` 暫時相容橋接，僅允許 `.120/.121` 來源，具備 check / apply / status / rollback 與 `2026-08-15` 到期日。 - K3s OTEL DaemonSet 新增 `file_storage`、filelog offset 持久化、exporter WAL、`queue_size=10000`、無限時 retry、60 秒 termination grace 與逐節點 rolling update。 +- Event Exporter 新增最小化 metadata-only RBAC，僅允許 `get` 事件引用的 Pod、Deployment、ReplicaSet、DaemonSet、Job、CronJob 與 Argo Application；明確不授權 Secret / ConfigMap 或任何寫入動作。 +- 修正 Event Exporter 的 durable write 缺口：舊平面 webhook JSON 雖收到 OTLP HTTP `200`，實際未產生 log；現改為標準 OTLP JSON envelope 與單一 receiver，避免未列出的 Normal 事件重複送出。 - 將現行 `0.96.0` image 由 K3s cache 直接匯入 Harbor，未對外下載；production 已 pin 至 `192.168.0.110:5000/awoooi/otel-collector-contrib@sha256:8edf386c...`。 - 新增獨立 ingestion verifier，以同一 run 綁定 K3s queue、`.188` bridge、`.110` listener 與 ClickHouse exact-marker 讀回。 **已跑驗證**： -- 精準 pytest：`7 passed`；三支 shell 的 `bash -n` 與 `git diff --check` 通過。 +- 精準 pytest：`11 passed`；三支 shell 的 `bash -n` 與 `git diff --check` 通過。 - `kubectl apply --dry-run=server`、exact live digest `file_storage` canary、Harbor digest pull canary 全數通過；canary 顯示 `Extension started: file_storage`、`Initializing new persistent queue`、`Everything is ready`。 - DaemonSet generation `4 -> 5` 完成逐節點 rollout；兩個 collector 均 `Ready`、`restarts=0`、queue capacity `10000`、queue size `0`。 - Synthetic OTLP 從 `.120 -> .188:24318 -> .110:4318 -> signoz_logs.logs_v2` 回讀 exact count `1`；verifier terminal 為 `queue_zero_failure_deltas_zero_window_15s`，兩個 production collector 後續連續超過 11 分鐘 exporter error `0`、queue `0`、send-failed `0`。 +- `P0-OBS-001-20260714T201500+0800-event-otlp-layout` synthetic Kubernetes Warning Event 在 ClickHouse 回讀 exact body `1`、結構化 reason / service attributes `1`；新 Pod `Ready=true`、`restarts=0`、metadata forbidden `0`、webhook delivery error `0`。 **仍維持**： - `setupCompleted=false` 與 SigNoz organization 未初始化仍是治理缺口；static override 與 `.188` 橋接是可回滾、有到期日的暫時控制，不是 Foundry 遷移完成。 diff --git a/k8s/observability/event-exporter.yaml b/k8s/observability/event-exporter.yaml index d49ca649a..0b62f5c78 100644 --- a/k8s/observability/event-exporter.yaml +++ b/k8s/observability/event-exporter.yaml @@ -24,6 +24,20 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "watch", "list"] + # P0-OBS-001: event-exporter enriches events by reading the referenced object. + # Keep this metadata-only: no list/watch here and never grant Secret access. + - apiGroups: [""] + resources: ["pods", "nodes", "services", "persistentvolumeclaims", "persistentvolumes"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get"] + - apiGroups: ["argoproj.io"] + resources: ["applications"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -50,9 +64,10 @@ data: logFormat: json route: routes: - # Warning 和 Error 事件: 全量保留 (RCA 關鍵資料) + # Warning/Error 全量保留；只排除已知高頻、低診斷價值的 Normal 事件。 + # 單一 receiver 避免未列出的 Normal 事件被兩條 route 重複送出。 - match: - - receiver: signoz-warning + - receiver: signoz-otlp drop: - type: "Normal" reason: "Scheduled" @@ -64,46 +79,65 @@ data: reason: "Created" - type: "Normal" reason: "Started" - # Normal 事件中的關鍵類型: 選擇性保留 - - match: - - receiver: signoz-normal - drop: - - type: "Warning" receivers: - - name: signoz-warning + - name: signoz-otlp webhook: endpoint: http://192.168.0.188:24318/v1/logs headers: Content-Type: application/json + # OTLP/HTTP JSON envelope。舊版的平面 JSON 雖會得到 HTTP 200， + # 但 collector 會把它當成空請求，無任何 durable log write。 layout: - severity: "{{ .Type }}" - body: "{{ .Message }}" - attributes: - k8s.event.reason: "{{ .Reason }}" - k8s.event.type: "{{ .Type }}" - k8s.event.action: "{{ .Action }}" - k8s.event.count: "{{ .Count }}" - k8s.namespace.name: "{{ .InvolvedObject.Namespace }}" - k8s.object.kind: "{{ .InvolvedObject.Kind }}" - k8s.object.name: "{{ .InvolvedObject.Name }}" - k8s.event.source.component: "{{ .Source.Component }}" - k8s.event.source.host: "{{ .Source.Host }}" - k8s.event.first_timestamp: "{{ .FirstTimestamp }}" - k8s.event.last_timestamp: "{{ .LastTimestamp }}" - - name: signoz-normal - webhook: - endpoint: http://192.168.0.188:24318/v1/logs - headers: - Content-Type: application/json - layout: - severity: "Normal" - body: "{{ .Message }}" - attributes: - k8s.event.reason: "{{ .Reason }}" - k8s.event.type: "{{ .Type }}" - k8s.namespace.name: "{{ .InvolvedObject.Namespace }}" - k8s.object.kind: "{{ .InvolvedObject.Kind }}" - k8s.object.name: "{{ .InvolvedObject.Name }}" + resourceLogs: + - resource: + attributes: + - key: service.name + value: + stringValue: kubernetes-event-exporter + - key: k8s.cluster.name + value: + stringValue: awoooi-k3s + scopeLogs: + - scope: + name: awoooi.k8s.events + logRecords: + - severityText: "{{ .Type }}" + body: + stringValue: "{{ .Message }}" + attributes: + - key: k8s.event.reason + value: + stringValue: "{{ .Reason }}" + - key: k8s.event.type + value: + stringValue: "{{ .Type }}" + - key: k8s.event.action + value: + stringValue: "{{ .Action }}" + - key: k8s.event.count + value: + stringValue: "{{ .Count }}" + - key: k8s.namespace.name + value: + stringValue: "{{ .InvolvedObject.Namespace }}" + - key: k8s.object.kind + value: + stringValue: "{{ .InvolvedObject.Kind }}" + - key: k8s.object.name + value: + stringValue: "{{ .InvolvedObject.Name }}" + - key: k8s.event.source.component + value: + stringValue: "{{ .Source.Component }}" + - key: k8s.event.source.host + value: + stringValue: "{{ .Source.Host }}" + - key: k8s.event.first_timestamp + value: + stringValue: "{{ .FirstTimestamp }}" + - key: k8s.event.last_timestamp + value: + stringValue: "{{ .LastTimestamp }}" --- apiVersion: apps/v1 kind: Deployment diff --git a/scripts/reboot-recovery/tests/test_event_exporter_metadata_rbac.py b/scripts/reboot-recovery/tests/test_event_exporter_metadata_rbac.py new file mode 100644 index 000000000..5fcfcf5c0 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_event_exporter_metadata_rbac.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[3] +MANIFEST = ROOT / "k8s" / "observability" / "event-exporter.yaml" + + +def test_event_exporter_has_minimal_metadata_enrichment_reads() -> None: + documents = [ + doc for doc in yaml.safe_load_all(MANIFEST.read_text(encoding="utf-8")) if doc + ] + role = next(doc for doc in documents if doc["kind"] == "ClusterRole") + rules = role["rules"] + + expected = { + ("", "pods"), + ("apps", "deployments"), + ("apps", "replicasets"), + ("apps", "daemonsets"), + ("batch", "jobs"), + ("batch", "cronjobs"), + ("argoproj.io", "applications"), + } + granted = { + (api_group, resource) + for rule in rules + for api_group in rule["apiGroups"] + for resource in rule["resources"] + if rule["verbs"] == ["get"] + } + assert expected <= granted + assert all(resource not in {"secrets", "configmaps"} for _, resource in granted) + + +def test_event_exporter_metadata_rules_never_write_or_delete() -> None: + documents = [ + doc for doc in yaml.safe_load_all(MANIFEST.read_text(encoding="utf-8")) if doc + ] + role = next(doc for doc in documents if doc["kind"] == "ClusterRole") + forbidden = {"create", "update", "patch", "delete", "deletecollection"} + assert not any(forbidden.intersection(rule["verbs"]) for rule in role["rules"]) + + +def _event_exporter_config() -> dict: + documents = [ + doc for doc in yaml.safe_load_all(MANIFEST.read_text(encoding="utf-8")) if doc + ] + configmap = next(doc for doc in documents if doc["kind"] == "ConfigMap") + return yaml.safe_load(configmap["data"]["config.yaml"]) + + +def test_event_exporter_webhook_emits_a_real_otlp_json_envelope() -> None: + config = _event_exporter_config() + receivers = config["receivers"] + assert [receiver["name"] for receiver in receivers] == ["signoz-otlp"] + + webhook = receivers[0]["webhook"] + assert webhook["endpoint"].endswith("/v1/logs") + assert webhook["headers"]["Content-Type"] == "application/json" + + resource_logs = webhook["layout"]["resourceLogs"] + log_record = resource_logs[0]["scopeLogs"][0]["logRecords"][0] + assert log_record["body"]["stringValue"] == "{{ .Message }}" + assert log_record["severityText"] == "{{ .Type }}" + attribute_keys = {item["key"] for item in log_record["attributes"]} + assert { + "k8s.event.reason", + "k8s.event.type", + "k8s.namespace.name", + "k8s.object.kind", + "k8s.object.name", + } <= attribute_keys + + +def test_event_exporter_has_one_route_and_drops_only_known_normal_noise() -> None: + config = _event_exporter_config() + routes = config["route"]["routes"] + assert len(routes) == 1 + assert routes[0]["match"] == [{"receiver": "signoz-otlp"}] + assert {item["type"] for item in routes[0]["drop"]} == {"Normal"} + assert {item["reason"] for item in routes[0]["drop"]} == { + "Scheduled", + "Pulling", + "Pulled", + "Created", + "Started", + }