wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(sre): enforce typed controlled automation
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 2m38s
Details
CD Pipeline / build-and-deploy (push) Failing after 24m19s
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details

Browse Source
This commit is contained in:
ogt
2026-07-16 17:32:01 +08:00
parent dfec0c9f7f
commit 541a32a9cb
130 changed files with 14849 additions and 1738 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
ops/alertmanager/alertmanager.yml
View File

@@ -4,7 +4,8 @@
# 修正後: https://awoooi.wooo.work/api/v1/webhooks/alertmanager (AWOOOI public API，複數，正確)
# 根據 feedback_alertmanager_awoooi_flow.md 鐵律
# 2026-04-09 Claude Sonnet 4.6 Asia/Taipei: 新增 Telegram Fallback (ADR-035)
# 架構: awoooi-webhook (主路徑) + telegram-direct (告警鏈路緊急旁路)
# 架構: awoooi-webhook (主路徑) + telegram-direct
# Agent99 對 AlertChainBroken_Alertmanager 採主動唯讀 poll，不是 receiver。
# telegram-direct 只允許處理 AWOOOI API / AlertChain 自身異常；一般 critical 必須走 AWOOOI API 治理鏈。
# 旁路目的地必須是 AwoooI SRE 戰情室；OPENCLAW_TG_CHAT_ID 只允許作缺值時的 fail-soft fallback。
# ⚠️ bot_token/chat_id 部署時由 secrets 替換，此檔為模板
@@ -37,7 +38,7 @@ route:
- alertname=~"AWOOOIApiDown|AlertmanagerDown|AlertChainBroken_.*|AlertChainUnhealthy|NoAlertsReceived2Hours"
receiver: 'telegram-direct'
group_wait: 10s
repeat_interval: 30m
repeat_interval: 2h
continue: true
- matchers:
- severity="critical"
@@ -72,14 +73,22 @@ receivers:
chat_id: SRE_GROUP_CHAT_ID_PLACEHOLDER
parse_mode: 'HTML'
message: |
🚨 <b>[Alertmanager Emergency]</b>
🚨 <b>SRE BYPASS｜告警主鏈異常</b>
{{ range .Alerts }}
├ <b>{{ .Labels.alertname }}</b>
├ 嚴重度: {{ .Labels.severity }}
├ 主機: {{ .Labels.host }}{{ .Labels.instance }}
└ {{ .Annotations.summary }}
<b>{{ .Labels.severity }}｜{{ .Labels.alertname }}</b>
├ 資產：{{ if .Labels.component }}{{ .Labels.component }}{{ else if .Labels.service }}{{ .Labels.service }}{{ else }}alert-chain{{ end }}
├ 狀態：{{ .Annotations.summary }}
└ 影響：AWOOOI 主鏈 delivery / automation receipt 可信度下降
{{ end }}
<i>⚠️ AWOOOI API / 告警鏈路可能異常，此為 SRE 戰情室緊急旁路</i>
🤖 <b>自動化證據</b>
├ 判斷：Prometheus deterministic rule（未呼叫 LLM）
├ Agent：Alertmanager emergency router
├ 動作：僅送緊急旁路；未宣稱已修復
├ Executor：none
└ Verifier：source-specific delivery metric + AWOOOI E2E receipt pending
<i>下一步：由 canonical typed route 建立受控修復與獨立驗證；同一 firing state 2 小時內不重送。</i>
send_resolved: false
inhibit_rules:

95
ops/monitoring/alerts-unified.yml
View File

@@ -15,6 +15,81 @@
groups:
# =========================================================================
# AI provider cloud-edge availability (source + missing-series truth)
# =========================================================================
- name: ollama_cloud_edge_availability
interval: 30s
rules:
- alert: OllamaGcpACloudEdgeUnavailable
expr: |
(probe_success{job="blackbox-http",provider="ollama_gcp_a",probe_origin="host110"} == 0)
or absent(probe_success{job="blackbox-http",provider="ollama_gcp_a",probe_origin="host110"})
for: 2m
labels:
severity: critical
layer: ai-provider
component: ollama-gcp-a
team: ai
alert_category: ollama_failover
auto_repair: "false"
annotations:
summary: "GCP-A Ollama cloud-edge probe missing or unavailable"
description: "Host110 cloud-edge probe failed or is absent. Per-request deterministic routing remains in the same AI-provider domain and attempts GCP-B next; no global provider mutation is authorized."
runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md"
- alert: OllamaGcpBCloudEdgeUnavailable
expr: |
(probe_success{job="blackbox-http",provider="ollama_gcp_b",probe_origin="host110"} == 0)
or absent(probe_success{job="blackbox-http",provider="ollama_gcp_b",probe_origin="host110"})
for: 2m
labels:
severity: critical
layer: ai-provider
component: ollama-gcp-b
team: ai
alert_category: ollama_failover
auto_repair: "false"
annotations:
summary: "GCP-B Ollama cloud-edge probe missing or unavailable"
description: "Host110 cloud-edge probe failed or is absent. Per-request deterministic routing remains in the same AI-provider domain and attempts host111 next; no global provider mutation is authorized."
runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md"
- alert: OllamaLocalUnavailable
expr: |
ollama_health_status{host="111"} == 0
for: 2m
labels:
severity: warning
layer: ai-provider
component: ollama-local
host: "111"
canonical_asset_id: "ai-provider:ollama_local"
team: ai
alert_category: ai_provider
auto_repair: "true"
annotations:
summary: "Host111 Ollama local provider unavailable or missing"
description: "K3s-side provider health receipt proves Host111 unhealthy. Route only to the Host111 LaunchAgent Ansible PlayBook; verify independently from host120 and host121, and never probe or repair retired Host110 Ollama."
runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md"
- alert: OllamaLocalHealthReceiptMissing
expr: absent(ollama_health_status{host="111"})
for: 2m
labels:
severity: warning
layer: observability
component: ollama-health-receipt
host: "111"
canonical_asset_id: "signal:ollama_local_health_receipt"
team: ai
alert_category: monitoring_drift
auto_repair: "false"
annotations:
summary: "Host111 Ollama health receipt is missing"
description: "The K3s-side health signal is absent, so Host111 health is unknown rather than failed. Create a monitoring drift work item; do not run the Host111 repair PlayBook from missing-series evidence."
runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md"
# =========================================================================
# Full-stack recovery scorecard recording rules
# =========================================================================
@@ -810,17 +885,23 @@ groups:
rules:
- alert: AlertChainBroken_Alertmanager
expr: |
sum(rate(awoooi_webhook_requests_total{source="alertmanager",status!="success"}[5m]))
/ sum(rate(awoooi_webhook_requests_total{source="alertmanager"}[5m])) > 0.1
for: 10m
(
sum(increase(alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m]))
/ clamp_min(sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])), 1)
) > 0.1
and
sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) >= 5
for: 5m
labels:
severity: critical
layer: k8s
layer: host
component: alertmanager
team: platform
auto_repair: "false"
auto_repair: "true"
alert_category: "alert_chain_health"
annotations:
summary: "Alertmanager Webhook 錯誤率 > 10%"
description: "告警鏈路可能斷裂，請執行 E2E 驗證"
summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%"
description: "只使用 receiver=awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。"
- alert: AlertChainBroken_Sentry
expr: |

95
ops/monitoring/alerts.yml
View File

@@ -15,6 +15,81 @@
groups:
# =========================================================================
# AI provider cloud-edge availability (source + missing-series truth)
# =========================================================================
- name: ollama_cloud_edge_availability
interval: 30s
rules:
- alert: OllamaGcpACloudEdgeUnavailable
expr: |
(probe_success{job="blackbox-http",provider="ollama_gcp_a",probe_origin="host110"} == 0)
or absent(probe_success{job="blackbox-http",provider="ollama_gcp_a",probe_origin="host110"})
for: 2m
labels:
severity: critical
layer: ai-provider
component: ollama-gcp-a
team: ai
alert_category: ollama_failover
auto_repair: "false"
annotations:
summary: "GCP-A Ollama cloud-edge probe missing or unavailable"
description: "Host110 cloud-edge probe failed or is absent. Per-request deterministic routing remains in the same AI-provider domain and attempts GCP-B next; no global provider mutation is authorized."
runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md"
- alert: OllamaGcpBCloudEdgeUnavailable
expr: |
(probe_success{job="blackbox-http",provider="ollama_gcp_b",probe_origin="host110"} == 0)
or absent(probe_success{job="blackbox-http",provider="ollama_gcp_b",probe_origin="host110"})
for: 2m
labels:
severity: critical
layer: ai-provider
component: ollama-gcp-b
team: ai
alert_category: ollama_failover
auto_repair: "false"
annotations:
summary: "GCP-B Ollama cloud-edge probe missing or unavailable"
description: "Host110 cloud-edge probe failed or is absent. Per-request deterministic routing remains in the same AI-provider domain and attempts host111 next; no global provider mutation is authorized."
runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md"
- alert: OllamaLocalUnavailable
expr: |
ollama_health_status{host="111"} == 0
for: 2m
labels:
severity: warning
layer: ai-provider
component: ollama-local
host: "111"
canonical_asset_id: "ai-provider:ollama_local"
team: ai
alert_category: ai_provider
auto_repair: "true"
annotations:
summary: "Host111 Ollama local provider unavailable or missing"
description: "K3s-side provider health receipt proves Host111 unhealthy. Route only to the Host111 LaunchAgent Ansible PlayBook; verify independently from host120 and host121, and never probe or repair retired Host110 Ollama."
runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md"
- alert: OllamaLocalHealthReceiptMissing
expr: absent(ollama_health_status{host="111"})
for: 2m
labels:
severity: warning
layer: observability
component: ollama-health-receipt
host: "111"
canonical_asset_id: "signal:ollama_local_health_receipt"
team: ai
alert_category: monitoring_drift
auto_repair: "false"
annotations:
summary: "Host111 Ollama health receipt is missing"
description: "The K3s-side health signal is absent, so Host111 health is unknown rather than failed. Create a monitoring drift work item; do not run the Host111 repair PlayBook from missing-series evidence."
runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md"
# =========================================================================
# 主機層告警 (host_alerts)
# =========================================================================
@@ -491,17 +566,23 @@ groups:
rules:
- alert: AlertChainBroken_Alertmanager
expr: |
sum(rate(awoooi_webhook_requests_total{source="alertmanager",status!="success"}[5m]))
/ sum(rate(awoooi_webhook_requests_total{source="alertmanager"}[5m])) > 0.1
for: 10m
(
sum(increase(alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m]))
/ clamp_min(sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])), 1)
) > 0.1
and
sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) >= 5
for: 5m
labels:
severity: critical
layer: k8s
layer: host
component: alertmanager
team: platform
auto_repair: "false"
auto_repair: "true"
alert_category: "alert_chain_health"
annotations:
summary: "Alertmanager Webhook 錯誤率 > 10%"
description: "告警鏈路可能斷裂，請執行 E2E 驗證"
summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%"
description: "只使用 receiver=awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。"
- alert: AlertChainBroken_Sentry
expr: |

25
ops/monitoring/coverage_report.py
View File

@@ -83,7 +83,12 @@ def analyze_services(registry: dict) -> dict:
"by_type": {},
"by_criticality": {},
"monitoring": {
"prometheus": {"covered": 0, "missing": []},
"prometheus": {
"covered": 0,
"missing": [],
"source_declared": 0,
"generated_unactivated": [],
},
"sentry": {"covered": 0, "missing": []},
"otel": {"covered": 0, "missing": []},
"langfuse": {"covered": 0, "missing": []},
@@ -109,7 +114,23 @@ def analyze_services(registry: dict) -> dict:
# 監控覆蓋
monitoring = svc.get("monitoring", {})
for key in ["prometheus", "sentry", "otel", "langfuse"]:
prometheus_source_declared = bool(
monitoring.get("prometheus")
or monitoring.get("blackbox_target_enabled")
or monitoring.get("agent_verifier")
)
if prometheus_source_declared:
stats["monitoring"]["prometheus"]["source_declared"] += 1
if monitoring.get("runtime_verified") is True:
stats["monitoring"]["prometheus"]["covered"] += 1
elif prometheus_source_declared:
stats["monitoring"]["prometheus"]["generated_unactivated"].append(
name
)
else:
stats["monitoring"]["prometheus"]["missing"].append(name)
for key in ["sentry", "otel", "langfuse"]:
if monitoring.get(key):
stats["monitoring"][key]["covered"] += 1
else:

62
ops/monitoring/generate_monitoring.py
View File

@@ -87,7 +87,7 @@ def generate_prometheus_scrape_configs(registry: dict) -> list[dict]:
scrape_configs.append(config)
elif svc.get("type") == "docker":
# Docker 直接 scrape
# Host service exposing native Prometheus metrics.
host = svc.get("host", "localhost")
port = svc.get("port", 8080)
config = {
@@ -96,7 +96,7 @@ def generate_prometheus_scrape_configs(registry: dict) -> list[dict]:
"targets": [f"{host}:{port}"],
"labels": {
"service": svc["name"],
"type": "docker",
"type": svc.get("type"),
"owner": svc.get("owner", "unknown"),
"criticality": svc.get("criticality", "P2")
}
@@ -144,25 +144,43 @@ def generate_blackbox_targets(registry: dict) -> list[dict]:
for svc in registry.get("services", []):
health_endpoint = svc.get("health_endpoint")
health_type = svc.get("health_type", "http")
monitoring = svc.get("monitoring", {})
if (
svc.get("type") == "ai-provider"
and monitoring.get("blackbox_target_enabled") is not True
):
continue
if health_endpoint and health_type == "http":
if svc.get("type") == "k8s-deployment":
# K8s 內部 DNS
url = f"http://{svc['name']}.{svc.get('namespace', 'default')}.svc.cluster.local:{svc.get('port', 8080)}{health_endpoint}"
elif svc.get("type") == "docker":
elif svc.get("type") in {"docker", "ai-provider"}:
host = svc.get("host", "localhost")
port = svc.get("port", 8080)
url = f"http://{host}:{port}{health_endpoint}"
else:
continue
labels = {
"service": svc["name"],
"criticality": svc.get("criticality", "P2"),
"owner": svc.get("owner", "unknown"),
}
if svc.get("type") == "ai-provider":
labels.update({
"provider": svc["name"].replace("-", "_"),
"probe_origin": monitoring.get("probe_origin", "unresolved"),
"boundary": monitoring.get("data_boundary", "unresolved"),
"coverage_state": monitoring.get(
"coverage_state",
"generated_unactivated",
),
})
targets.append({
"url": url,
"labels": {
"service": svc["name"],
"criticality": svc.get("criticality", "P2"),
"owner": svc.get("owner", "unknown")
}
"labels": labels,
})
# API 端點
@@ -191,6 +209,9 @@ def validate_coverage(registry: dict) -> dict:
report = {
"total_services": 0,
"monitored_services": 0,
"source_declared_services": 0,
"generated_unactivated": [],
"runtime_verified_services": 0,
"coverage_percent": 0.0,
"missing_prometheus": [],
"missing_health_endpoint": [],
@@ -202,8 +223,18 @@ def validate_coverage(registry: dict) -> dict:
report["total_services"] += 1
monitoring = svc.get("monitoring", {})
if monitoring.get("prometheus"):
source_declared = bool(
monitoring.get("prometheus")
or monitoring.get("blackbox_target_enabled")
or monitoring.get("agent_verifier")
)
if source_declared:
report["source_declared_services"] += 1
if monitoring.get("runtime_verified") is True:
report["monitored_services"] += 1
report["runtime_verified_services"] += 1
elif source_declared:
report["generated_unactivated"].append(svc["name"])
else:
report["missing_prometheus"].append(svc["name"])
@@ -258,8 +289,17 @@ def print_coverage_report(report: dict):
print(" AWOOOI Monitoring Coverage Report")
print("=" * 60)
print(f"\n Total Services: {report['total_services']}")
print(f" Monitored: {report['monitored_services']}")
print(f" Coverage: {report['coverage_percent']}%")
print(f" Source Declared: {report['source_declared_services']}")
print(f" Runtime Verified: {report['runtime_verified_services']}")
print(f" Runtime Coverage: {report['coverage_percent']}%")
if report["generated_unactivated"]:
print(
f"\n Generated/Declared But Runtime Pending "
f"({len(report['generated_unactivated'])}):"
)
for svc in report["generated_unactivated"]:
print(f" - {svc}")
if report["missing_prometheus"]:
print(f"\n Missing Prometheus Monitoring ({len(report['missing_prometheus'])}):")

23
ops/monitoring/generated/blackbox-targets-generated.yaml
View File

@@ -17,27 +17,30 @@
criticality: P0
owner: devops-team
service: prometheus
url: http://prometheus.monitoring.svc.cluster.local:9090/-/ready
url: http://192.168.0.110:9090/-/ready
- labels:
criticality: P0
owner: devops-team
service: alertmanager
url: http://alertmanager.monitoring.svc.cluster.local:9093/-/ready
url: http://192.168.0.110:9093/-/ready
- labels:
boundary: cloud_edge_availability_only
coverage_state: source_candidate_runtime_pending
criticality: P0
owner: ai-team
probe_origin: host110
provider: ollama_gcp_a
service: ollama-gcp-a
url: http://192.168.0.110:11435/api/tags
url: http://34.143.170.20:11434/api/tags
- labels:
boundary: cloud_edge_availability_only
coverage_state: source_candidate_runtime_pending
criticality: P0
owner: ai-team
probe_origin: host110
provider: ollama_gcp_b
service: ollama-gcp-b
url: http://192.168.0.110:11436/api/tags
- labels:
criticality: P0
owner: ai-team
service: ollama-local
url: http://192.168.0.110:11437/api/tags
url: http://34.21.145.224:11434/api/tags
- labels:
criticality: P0
owner: ai-team
@@ -47,7 +50,7 @@
criticality: P2
owner: devops-team
service: signoz-ui
url: http://192.168.0.188:3301/
url: http://192.168.0.110:8080/api/v1/health
- labels:
criticality: P1
owner: devops-team

171
ops/monitoring/generated/prometheus-scrape-generated.yaml
View File

@@ -67,15 +67,151 @@ scrape_configs:
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: alertmanager
- job_name: argocd-applicationset-controller
kubernetes_sd_configs:
- namespaces:
names:
- monitoring
- argocd
role: pod
relabel_configs:
- action: keep
regex: alertmanager
regex: argocd-applicationset-controller
source_labels:
- __meta_kubernetes_pod_label_app
- source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: argocd-dex-server
kubernetes_sd_configs:
- namespaces:
names:
- argocd
role: pod
relabel_configs:
- action: keep
regex: argocd-dex-server
source_labels:
- __meta_kubernetes_pod_label_app
- source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: argocd-notifications-controller
kubernetes_sd_configs:
- namespaces:
names:
- argocd
role: pod
relabel_configs:
- action: keep
regex: argocd-notifications-controller
source_labels:
- __meta_kubernetes_pod_label_app
- source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: argocd-redis
kubernetes_sd_configs:
- namespaces:
names:
- argocd
role: pod
relabel_configs:
- action: keep
regex: argocd-redis
source_labels:
- __meta_kubernetes_pod_label_app
- source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: argocd-repo-server
kubernetes_sd_configs:
- namespaces:
names:
- argocd
role: pod
relabel_configs:
- action: keep
regex: argocd-repo-server
source_labels:
- __meta_kubernetes_pod_label_app
- source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: awoooi-api
kubernetes_sd_configs:
- namespaces:
names:
- awoooi-dev
role: pod
relabel_configs:
- action: keep
regex: awoooi-api
source_labels:
- __meta_kubernetes_pod_label_app
- source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: kube-state-metrics
kubernetes_sd_configs:
- namespaces:
names:
- kube-state-metrics
role: pod
relabel_configs:
- action: keep
regex: kube-state-metrics
source_labels:
- __meta_kubernetes_pod_label_app
- source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: event-exporter
kubernetes_sd_configs:
- namespaces:
names:
- observability
role: pod
relabel_configs:
- action: keep
regex: event-exporter
source_labels:
- __meta_kubernetes_pod_label_app
- source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: velero
kubernetes_sd_configs:
- namespaces:
names:
- velero
role: pod
relabel_configs:
- action: keep
regex: velero
source_labels:
- __meta_kubernetes_pod_label_app
- source_labels:
@@ -84,17 +220,6 @@ scrape_configs:
- source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- job_name: ollama
static_configs:
- labels:
criticality: P0
owner: ai-team
service: ollama
type: docker
targets:
- 192.168.0.110:11435
- 192.168.0.110:11436
- 192.168.0.110:11437
- job_name: openclaw
static_configs:
- labels:
@@ -122,24 +247,6 @@ scrape_configs:
type: docker
targets:
- 192.168.0.188:5432
- job_name: signoz-collector
static_configs:
- labels:
criticality: P1
owner: devops-team
service: signoz-collector
type: docker
targets:
- 192.168.0.188:24317
- job_name: signoz-ui
static_configs:
- labels:
criticality: P2
owner: devops-team
service: signoz-ui
type: docker
targets:
- 192.168.0.188:3301
- job_name: clickhouse
static_configs:
- labels:

19
ops/monitoring/grafana/dashboards/ollama_failover.json
View File

@@ -35,7 +35,7 @@
"annotations": {
"list": []
},
"description": "Ollama 容災監控 — 可用性、推理延遲、AI 路由分布、Failover/Recovery 觸發 | P2.3 2026-04-26 台北時區",
"description": "Ollama 容災監控 — GCP-A/B cloud-edge probe truth、推理延遲、AI 路由分布、Failover/Recovery 觸發。Host111 由 K3s host120/121 independent verifier 補證，尚未取得 runtime receipt 時不得假綠。",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
@@ -81,14 +81,23 @@
},
"textMode": "auto"
},
"title": "Ollama 可用性",
"description": "up{job=~\"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111\"} × 100\n- 綠色 100% = 主機在線\n- 紅色 0% = 主機離線（容災應已觸發）\n\n資料來源: Prometheus scrape job ollama_gcp_a / ollama_gcp_b / ollama_local",
"title": "GCP-A/B Cloud Edge 可用性",
"description": "GCP-A/B 僅顯示 host110 blackbox /api/tags 的 probe_success；absent series 明確轉為 0，避免 blackbox-http 其他網站健康時假綠。\n\nHost111 不從已退役的 host110 proxy/probe；狀態為 independent K3s verifier pending，需 host120/121 verifier runtime receipt 才能另行標綠。",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "up{job=~\"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111\"} * 100",
"legendFormat": "{{ job }}",
"expr": "100 * (max by (provider) (probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_a\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}) or label_replace(0 * absent(probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_a\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}), \"provider\", \"ollama_gcp_a\", \"\", \"\"))",
"legendFormat": "GCP-A",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "100 * (max by (provider) (probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_b\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}) or label_replace(0 * absent(probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_b\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}), \"provider\", \"ollama_gcp_b\", \"\", \"\"))",
"legendFormat": "GCP-B",
"refId": "B"
}
],
"type": "stat"

30
ops/monitoring/ollama_health_rules.yaml
View File

@@ -5,6 +5,15 @@
# 部署目標: 與 alerts-unified.yml 一起部署到 192.168.0.110:/home/wooo/monitoring/alerts.yml
# 部署方式: 手動合併至 alerts-unified.yml，或 scripts/ops/deploy-alerts.sh 支援多檔時直接引用
#
# Legacy policy sunset (2026-07-16):
# superseded_by=global_product_governance_v2
# OllamaInstanceDown 已移除。它依賴不存在的 ollama_gcp_a / ollama_gcp_b /
# ollama_local / ollama_111 native scrape jobs，會在 series 缺失時假綠。
# Active source truth is ops/monitoring/alerts-unified.yml:
# - OllamaGcpACloudEdgeUnavailable
# - OllamaGcpBCloudEdgeUnavailable
# Host111 不從退役 host110 probe；由 K3s host120/121 independent verifier 補證。
#
# 標籤規範 (對齊 alerts-unified.yml):
# layer: ai-provider
# team: ai
@@ -26,27 +35,6 @@ groups:
interval: 30s
rules:
# -----------------------------------------------------------------------
# 🔴 [ACTIVE] Ollama 主機離線
# metric: up{job=~"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111"}
# 前置條件: Prometheus scrape job 命名對齊 ADR-110 provider pool
# (設定位於 ops/monitoring/generated/prometheus-scrape-generated.yaml)
# -----------------------------------------------------------------------
- alert: OllamaInstanceDown
expr: up{job=~"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111"} == 0
for: 2m
labels:
severity: critical
layer: ai-provider
team: ai
auto_repair: "false"
alert_category: "ollama_failover"
annotations:
summary: "Ollama {{ $labels.job }} 離線 ({{ $labels.instance }})"
description: "Prometheus 探測 Ollama {{ $labels.job }} 失敗超過 2 分鐘。預期容災應已觸發，路由已切 Gemini。"
runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md#ollama-instance-down"
action: "curl http://34.143.170.20:11434/api/tags（GCP-A）或 curl http://34.21.145.224:11434/api/tags（GCP-B）或 ssh wooo@192.168.0.111 'systemctl status ollama'（Local 後備）"
# -----------------------------------------------------------------------
# 🟡 [ACTIVE] Failover 觸發頻率過高
# metric: ollama_failover_triggered_total{from_provider,to_provider}

117
ops/monitoring/service-registry.yaml
View File

@@ -100,12 +100,13 @@ services:
# --- Prometheus ---
- name: prometheus
type: k8s-deployment
namespace: monitoring
type: docker
host: 192.168.0.110
port: 9090
health_endpoint: /-/ready
monitoring:
prometheus: false # 自己監控自己會循環
prometheus: true
prometheus_scrape: false # canonical self-scrape job is managed explicitly
sentry: false
alerts:
- service_down
@@ -114,15 +115,21 @@ services:
# --- Alertmanager ---
- name: alertmanager
type: k8s-deployment
namespace: monitoring
type: docker
host: 192.168.0.110
port: 9093
health_endpoint: /-/ready
monitoring:
prometheus: true
prometheus_scrape: false # alertmanager-runtime job owns exact delivery truth
sentry: false
alerts:
- service_down
- delivery_error_rate
auto_repair:
enabled: true
actions:
- ansible:110-alertmanager-delivery-recovery
owner: devops-team
criticality: P0
@@ -241,17 +248,23 @@ services:
criticality: P1
# =============================================================================
# Docker 容器 (192.168.0.188 - AI/Web 中心)
# AI Provider 與 Docker 服務
# =============================================================================
# --- Ollama LLM ---
- name: ollama
type: docker
host: 192.168.0.188
# --- Ollama provider pool（host110 proxy 已於 2026-07-16 退役）---
- name: ollama-gcp-a
type: ai-provider
host: 34.143.170.20
port: 11434
health_endpoint: /api/tags
monitoring:
prometheus: true
prometheus: false
prometheus_scrape: false
blackbox_target_enabled: true
coverage_state: source_candidate_runtime_pending
probe_origin: host110
data_boundary: cloud_edge_availability_only
runtime_verified: false
sentry: false
otel: false
alerts:
@@ -261,7 +274,63 @@ services:
auto_repair:
enabled: true
actions:
- restart_container
- route_same_domain_next_hop
owner: ai-team
criticality: P0
- name: ollama-gcp-b
type: ai-provider
host: 34.21.145.224
port: 11434
health_endpoint: /api/tags
monitoring:
prometheus: false
prometheus_scrape: false
blackbox_target_enabled: true
coverage_state: source_candidate_runtime_pending
probe_origin: host110
data_boundary: cloud_edge_availability_only
runtime_verified: false
sentry: false
otel: false
alerts:
- service_down
- inference_timeout
- model_load_failed
auto_repair:
enabled: true
actions:
- route_same_domain_next_hop
owner: ai-team
criticality: P0
- name: ollama-local
type: ai-provider
host: 192.168.0.111
port: 11434
health_endpoint: /api/tags
runtime_manager: launchagent
runtime_unit: com.momo.ollama111-allow-proxy
monitoring:
prometheus: false
prometheus_scrape: false
blackbox_target_enabled: false
agent_verifier: true
coverage_state: k3s_path_verifier_runtime_pending
probe_origins:
- host120
- host121
runtime_verified: false
sentry: false
otel: false
alerts:
- service_down
- inference_timeout
- model_load_failed
auto_repair:
enabled: true
actions:
- ansible:111-ollama-fallback
owner: ai-team
criticality: P0
@@ -425,24 +494,6 @@ services:
owner: ai-team
criticality: P2
# --- GitHub Actions Runner ---
- name: github-runner
type: systemd
host: 192.168.0.110
service_name: actions.runner.owenhytsai-awoooi.awoooi-110.service
monitoring:
prometheus: true
sentry: false
alerts:
- runner_offline
- job_stuck
auto_repair:
enabled: true
actions:
- restart_service
owner: devops-team
criticality: P0
# =============================================================================
# 主機節點
# =============================================================================
@@ -759,12 +810,6 @@ alert_templates:
severity: warning
auto_repair: switch_model
runner_offline:
expr: 'github_runner_status == 0'
for: 5m
severity: critical
auto_repair: restart_service
# --- NVIDIA Nemotron 告警 (Phase 20) ---
# 2026-03-29 ogt: ADR-036 新增
circuit_breaker_open:

15
ops/monitoring/validate_coverage.py
View File

@@ -181,7 +181,20 @@ def validate_registry(registry: dict) -> ValidationResult:
total = len(services) if services else 1
coverage = {
'prometheus': sum(1 for s in services if s.get('monitoring', {}).get('prometheus')) / total,
'prometheus_source_declared': sum(
1
for s in services
if (
s.get('monitoring', {}).get('prometheus')
or s.get('monitoring', {}).get('blackbox_target_enabled')
or s.get('monitoring', {}).get('agent_verifier')
)
) / total,
'prometheus_runtime_verified': sum(
1
for s in services
if s.get('monitoring', {}).get('runtime_verified') is True
) / total,
'sentry': sum(1 for s in services if s.get('monitoring', {}).get('sentry')) / total,
'otel': sum(1 for s in services if s.get('monitoring', {}).get('otel')) / total,
'alerts': sum(1 for s in services if s.get('alerts')) / total,

127
ops/nginx/deploy-ollama-proxy-110.sh Executable file → Normal file
View File

@@ -1,121 +1,10 @@
#!/bin/bash
# GCP Ollama Nginx Proxy 部署腳本 (110 手動執行)
# ADR-110 三層容災 — 讓 K3s 透過內網存取 GCP Ollama
# 執行: ssh wooo@192.168.0.110 'sudo bash -s' < deploy-ollama-proxy-110.sh
#!/usr/bin/env bash
# superseded_by=global_product_governance_v2
# owner=AssetIdentity; expiry=2026-07-16
# replacement=direct GCP-A/GCP-B/host111 provider identities
# verifier=apps/api/tests/test_ollama_prod_manifest_order.py
set -euo pipefail
echo "🚀 部署 GCP Ollama Nginx Proxy (110)..."
# 配置內容
NGINX_CONF="/etc/nginx/sites-enabled/110-ollama-proxy.conf"
# 備份現有配置
if [ -f "$NGINX_CONF" ]; then
echo "📦 備份現有配置..."
cp "$NGINX_CONF" "${NGINX_CONF}.backup.$(date +%Y%m%d%H%M%S)"
fi
# 寫入 nginx 配置
echo "📝 寫入 nginx 配置..."
cat > "$NGINX_CONF" << 'EOF'
# 110 Ollama GCP Proxy — ADR-110 三層容災
# 讓 K3s 叢集內可透過內網 110 存取 GCP 外網 Ollama
# ============================================================
# Ollama GCP-A Primary (port 11435 → 34.143.170.20:11434)
# ============================================================
server {
listen 11435;
listen [::]:11435;
server_name _;
access_log /var/log/nginx/ollama-gcp-a-access.log;
error_log /var/log/nginx/ollama-gcp-a-error.log warn;
location / {
proxy_pass http://34.143.170.20:11434;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# Ollama 推理可能較慢，給較長超時
proxy_connect_timeout 10s;
proxy_send_timeout 300s;
proxy_read_timeout 300s;
# 支援 streaming response
proxy_buffering off;
proxy_cache off;
}
# 健康檢查端點
location /nginx-health {
access_log off;
return 200 "Ollama GCP-A Proxy OK\n";
add_header Content-Type text/plain;
}
}
# ============================================================
# Ollama GCP-B Secondary (port 11436 → 34.21.145.224:11434)
# ============================================================
server {
listen 11436;
listen [::]:11436;
server_name _;
access_log /var/log/nginx/ollama-gcp-b-access.log;
error_log /var/log/nginx/ollama-gcp-b-error.log warn;
location / {
proxy_pass http://34.21.145.224:11434;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 10s;
proxy_send_timeout 300s;
proxy_read_timeout 300s;
proxy_buffering off;
proxy_cache off;
}
location /nginx-health {
access_log off;
return 200 "Ollama GCP-B Proxy OK\n";
add_header Content-Type text/plain;
}
}
EOF
# 測試 nginx 配置
echo "🧪 測試 nginx 配置..."
nginx -t
# 重載 nginx
echo "🔄 重載 nginx..."
systemctl reload nginx
# 驗證端口監聽
echo "🔍 驗證端口監聽..."
sleep 2
ss -tlnp | grep -E '11435|11436' || true
# 本地測試
echo "🌐 本地測試 proxy..."
echo "測試 GCP-A proxy (11435)..."
curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:11435/api/tags || echo "連線失敗"
echo ""
echo "測試 GCP-B proxy (11436)..."
curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:11436/api/tags || echo "連線失敗"
echo ""
echo "✅ 部署完成！"
echo ""
echo "下一步："
echo "1. 從 K3s node 測試: curl http://192.168.0.110:11435/api/tags"
echo "2. 修改 K8s ConfigMap 指向 110:11435/11436"
echo "3. 重啟 awoooi-api deployment"
printf '%s\n' \
'BLOCKED: host110 Ollama transport is retired; this script cannot deploy it.' >&2
exit 78