#!/usr/bin/env python3
"""
generate_monitoring.py — 監控覆蓋率自動發現
Phase O-5 Wave C.1 (2026-04-02 ogt)

功能:
  1. 查詢 Prometheus targets API，取得全量 scrape 狀態
  2. 掃描 K8s Services，找出未被監控的服務
  3. 輸出覆蓋率報告 (JSON + 人可讀格式)

用法:
  python3 scripts/generate_monitoring.py
  python3 scripts/generate_monitoring.py --json
  python3 scripts/generate_monitoring.py --check  # CI mode: exit 1 if coverage < threshold
"""

import argparse
import json
import os
import subprocess
import sys
import time
from datetime import datetime
from typing import Callable
from urllib.error import HTTPError, URLError
from urllib.request import urlopen

# ============================================================
# 設定
# ============================================================

PROMETHEUS_URL = "http://192.168.0.110:9090"
COVERAGE_THRESHOLD = 70  # CI 模式: 覆蓋率低於此值則 exit 1
DEFAULT_STABILIZATION_ATTEMPTS = 3
DEFAULT_STABILIZATION_SLEEP_SECONDS = 10.0

# 已知服務清單 (job名稱 → 說明)
EXPECTED_JOBS = {
    "awoooi-api": "AWOOOI API (K8s)",
    "clawbot": "OpenClaw 188:8088",
    "node-exporter-110": "Node Exporter 110",
    "node-exporter-112": "Node Exporter 112 (Kali)",
    "node-exporter-188": "Node Exporter 188",
    "cadvisor-110": "cAdvisor 110",
    "prometheus": "Prometheus self-scrape",
    "blackbox-http": "Blackbox HTTP probe",
    "blackbox-tcp": "Blackbox TCP probe",
    "gitea-native": "Gitea native metrics",
}

# Jobs alone are not sufficient evidence for multi-target scrape jobs.  A
# healthy website target can keep ``blackbox-http`` green while both AI
# provider probes are absent.  Keep the exact cloud-edge identities here so
# source/runtime reconciliation fails closed on missing, down, or duplicated
# provider targets.
EXPECTED_TARGETS = {
    "ollama_gcp_a_cloud_edge": {
        "description": "GCP-A Ollama cloud-edge /api/tags from host110",
        "labels": {
            "job": "blackbox-http",
            "provider": "ollama_gcp_a",
            "probe_origin": "host110",
            "boundary": "cloud_edge_availability_only",
            "instance": "http://34.143.170.20:11434/api/tags",
        },
    },
    "ollama_gcp_b_cloud_edge": {
        "description": "GCP-B Ollama cloud-edge /api/tags from host110",
        "labels": {
            "job": "blackbox-http",
            "provider": "ollama_gcp_b",
            "probe_origin": "host110",
            "boundary": "cloud_edge_availability_only",
            "instance": "http://34.21.145.224:11434/api/tags",
        },
    },
}

# GitHub is globally frozen.  A legacy exporter still appearing UP must be
# reported as policy drift, not counted as harmless surplus coverage.
FORBIDDEN_JOBS = {
    "github-actions": "retired GitHub exporter must be removed from production",
}

# 允許 down 的 target (已知問題，不影響覆蓋率計算)
KNOWN_DOWN_TARGETS = {
    "federation-k8s": "K8s federation — SigNoz 內部 Prometheus，非外部暴露",
    "kube-state-metrics": "kube-state-metrics NodePort 30180 — 僅 OTEL Collector 內部存取",
    "node-exporter-120": "node-exporter 120 — K8s master 節點防火牆規則",
    "node-exporter-121": "node-exporter 121 — K8s worker 節點防火牆規則",
}


def _int_env(name: str, default: int) -> int:
    try:
        return max(1, int(os.environ.get(name, default)))
    except ValueError:
        return default


def _float_env(name: str, default: float) -> float:
    try:
        return max(0.0, float(os.environ.get(name, default)))
    except ValueError:
        return default


def get_prometheus_targets() -> dict:
    """查詢 Prometheus targets API"""
    try:
        with urlopen(f"{PROMETHEUS_URL}/api/v1/targets", timeout=10) as resp:
            payload = json.loads(resp.read().decode("utf-8"))
        return payload["data"]
    except (HTTPError, URLError, TimeoutError, json.JSONDecodeError, KeyError) as e:
        print(f"❌ 無法連接 Prometheus ({PROMETHEUS_URL}): {e}", file=sys.stderr)
        sys.exit(1)


def get_k8s_services() -> list[dict]:
    """查詢 K8s services (需要 kubectl)"""
    try:
        result = subprocess.run(
            ["kubectl", "get", "services", "--all-namespaces", "-o", "json"],
            capture_output=True,
            text=True,
            timeout=15,
        )
        if result.returncode != 0:
            return []
        data = json.loads(result.stdout)
        return data.get("items", [])
    except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError):
        return []


def analyze_targets(targets_data: dict) -> dict:
    """分析 targets 狀態"""
    active = targets_data.get("activeTargets", [])

    jobs: dict[str, dict] = {}
    for t in active:
        job = t["labels"].get("job", "unknown")
        instance = t["labels"].get("instance", "?")
        health = t["health"]

        if job not in jobs:
            jobs[job] = {"up": [], "down": [], "unknown": []}
        jobs[job][health].append(instance)

    return jobs


def analyze_expected_targets(targets_data: dict) -> dict:
    """Resolve exact multi-target identities without allowing job-level false green."""
    active = targets_data.get("activeTargets", [])
    target_status: dict[str, dict] = {}

    for target_id, expected in EXPECTED_TARGETS.items():
        expected_labels = expected["labels"]
        matches = [
            target
            for target in active
            if all(
                target.get("labels", {}).get(label) == value
                for label, value in expected_labels.items()
            )
        ]
        health_values = [str(target.get("health", "unknown")) for target in matches]
        if not matches:
            state = "missing"
        elif len(matches) != 1:
            state = "ambiguous"
        elif health_values == ["up"]:
            state = "up"
        else:
            state = "down"

        target_status[target_id] = {
            "description": expected["description"],
            "expected_labels": expected_labels,
            "match_count": len(matches),
            "health": health_values,
            "state": state,
        }

    return target_status


def build_report(jobs: dict) -> dict:
    """建立覆蓋率報告"""
    total_jobs = len(jobs)
    up_jobs = sum(1 for j in jobs.values() if j["up"] and not j["down"])
    partial_jobs = sum(1 for j in jobs.values() if j["up"] and j["down"])
    down_jobs = sum(1 for j in jobs.values() if not j["up"] and j["down"])

    # 只計算非 known_down 的問題
    real_down_jobs = {
        job: data
        for job, data in jobs.items()
        if not data["up"] and job not in KNOWN_DOWN_TARGETS
    }

    expected_covered = sum(1 for j in EXPECTED_JOBS if j in jobs and jobs[j]["up"])
    coverage_pct = round(expected_covered / len(EXPECTED_JOBS) * 100, 1)
    forbidden_jobs_present = [job for job in FORBIDDEN_JOBS if job in jobs]

    return {
        "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
        "prometheus_url": PROMETHEUS_URL,
        "summary": {
            "total_jobs": total_jobs,
            "up_jobs": up_jobs,
            "partial_jobs": partial_jobs,
            "down_jobs": down_jobs,
            "real_down_jobs": len(real_down_jobs),
            "expected_coverage_pct": coverage_pct,
            "forbidden_jobs_present": len(forbidden_jobs_present),
        },
        "jobs": jobs,
        "expected_jobs": EXPECTED_JOBS,
        "known_down": KNOWN_DOWN_TARGETS,
        "real_down_jobs": list(real_down_jobs.keys()),
        "missing_expected": [j for j in EXPECTED_JOBS if j not in jobs],
        "forbidden_jobs": FORBIDDEN_JOBS,
        "forbidden_jobs_present": forbidden_jobs_present,
    }


def build_report_from_targets(targets_data: dict) -> dict:
    """從 Prometheus targets API payload 建立覆蓋率報告"""
    report = build_report(analyze_targets(targets_data))
    expected_targets = analyze_expected_targets(targets_data)
    missing_expected_targets = [
        target_id
        for target_id, target in expected_targets.items()
        if target["state"] == "missing"
    ]
    down_expected_targets = [
        target_id
        for target_id, target in expected_targets.items()
        if target["state"] == "down"
    ]
    ambiguous_expected_targets = [
        target_id
        for target_id, target in expected_targets.items()
        if target["state"] == "ambiguous"
    ]
    report["expected_targets"] = expected_targets
    report["missing_expected_targets"] = missing_expected_targets
    report["down_expected_targets"] = down_expected_targets
    report["ambiguous_expected_targets"] = ambiguous_expected_targets
    report["summary"].update(
        {
            "expected_target_count": len(expected_targets),
            "expected_targets_up": sum(
                target["state"] == "up" for target in expected_targets.values()
            ),
            "expected_targets_missing": len(missing_expected_targets),
            "expected_targets_down": len(down_expected_targets),
            "expected_targets_ambiguous": len(ambiguous_expected_targets),
        }
    )
    return report


def report_needs_stabilization(report: dict) -> bool:
    """是否需要重查，避免 post-deploy 瞬間 scrape 狀態造成 false red."""
    return bool(
        report["real_down_jobs"]
        or report["missing_expected"]
        or report["missing_expected_targets"]
        or report["down_expected_targets"]
        or report["ambiguous_expected_targets"]
        or report["forbidden_jobs_present"]
    )


def stabilization_reason(report: dict) -> str:
    parts: list[str] = []
    if report["real_down_jobs"]:
        parts.append(f"real_down={','.join(report['real_down_jobs'])}")
    if report["missing_expected"]:
        parts.append(f"missing_expected={','.join(report['missing_expected'])}")
    if report["missing_expected_targets"]:
        parts.append(
            "missing_expected_targets="
            f"{','.join(report['missing_expected_targets'])}"
        )
    if report["down_expected_targets"]:
        parts.append(
            f"down_expected_targets={','.join(report['down_expected_targets'])}"
        )
    if report["ambiguous_expected_targets"]:
        parts.append(
            "ambiguous_expected_targets="
            f"{','.join(report['ambiguous_expected_targets'])}"
        )
    if report["forbidden_jobs_present"]:
        parts.append(
            f"forbidden_jobs_present={','.join(report['forbidden_jobs_present'])}"
        )
    return "; ".join(parts) if parts else "stable"


def build_stabilized_report(
    fetch_targets: Callable[[], dict],
    attempts: int,
    sleep_seconds: float,
    emit_status: bool = True,
) -> dict:
    """重查 Prometheus targets，讓 CI gate 避開 rollout/scrape freshness 瞬間值."""
    attempts = max(1, attempts)
    sleep_seconds = max(0.0, sleep_seconds)

    report: dict | None = None
    for attempt in range(1, attempts + 1):
        report = build_report_from_targets(fetch_targets())
        needs_retry = report_needs_stabilization(report)
        status = "stable"
        if needs_retry and attempt < attempts:
            status = "retrying"
        elif needs_retry:
            status = "failed"
        elif attempt > 1:
            status = "cleared"

        report["stabilization"] = {
            "attempt": attempt,
            "attempts": attempts,
            "sleep_seconds": sleep_seconds,
            "status": status,
            "reason": stabilization_reason(report),
        }

        if not needs_retry or attempt == attempts:
            if emit_status and attempt > 1 and not needs_retry:
                print(
                    "✅ Prometheus target stabilization cleared transient coverage drift",
                    file=sys.stderr,
                )
            return report

        if emit_status:
            print(
                "⏳ Prometheus target stabilization "
                f"{attempt}/{attempts}: {stabilization_reason(report)}",
                file=sys.stderr,
            )
        time.sleep(sleep_seconds)

    if report is None:
        raise RuntimeError("monitoring report stabilization did not run")
    return report


def print_human_report(report: dict) -> None:
    """輸出人可讀格式報告"""
    s = report["summary"]
    print(f"\n{'='*60}")
    print("  AWOOOI 監控覆蓋率報告")
    print(f"  生成時間: {report['generated_at']}")
    print(f"{'='*60}")
    print("\n📊 總覽")
    print(f"  Jobs 總數:        {s['total_jobs']}")
    print(f"  全部 UP:          {s['up_jobs']}")
    print(f"  部分 UP:          {s['partial_jobs']}")
    print(f"  全部 DOWN:        {s['down_jobs']}")
    print(f"  真實問題 (非已知): {s['real_down_jobs']}")
    print(f"  預期覆蓋率:       {s['expected_coverage_pct']}% ({COVERAGE_THRESHOLD}% 門檻)")
    print(
        "  精確 Target:      "
        f"{s['expected_targets_up']}/{s['expected_target_count']} UP "
        f"(missing={s['expected_targets_missing']}, "
        f"down={s['expected_targets_down']}, "
        f"ambiguous={s['expected_targets_ambiguous']})"
    )
    print(f"  禁止 Job 漂移:   {s['forbidden_jobs_present']}")

    print("\n✅ 預期服務狀態")
    for job, desc in report["expected_jobs"].items():
        jobs = report["jobs"]
        if job not in jobs:
            status = "❌ 缺失"
        elif jobs[job]["up"] and not jobs[job]["down"]:
            status = "✅ UP"
        elif jobs[job]["up"]:
            status = f"⚠️  部分 UP ({len(jobs[job]['up'])} up, {len(jobs[job]['down'])} down)"
        else:
            status = "❌ DOWN"
        print(f"  {status:<30} {job:<25} {desc}")

    known_down_present = [
        (job, reason)
        for job, reason in report["known_down"].items()
        if job in report["jobs"] and report["jobs"][job]["down"]
    ]
    if known_down_present:
        print("\n⚠️  已知 DOWN (不影響覆蓋率)")
        for job, reason in known_down_present:
            print(f"  {job:<30} {reason}")

    if report["real_down_jobs"]:
        print("\n🔴 需處理的 DOWN targets")
        for job in report["real_down_jobs"]:
            instances = report["jobs"][job].get("down", [])
            print(f"  {job}: {', '.join(instances)}")

    if report["missing_expected"]:
        print("\n🔴 缺少預期服務監控")
        for job in report["missing_expected"]:
            print(f"  {job}: {report['expected_jobs'][job]}")

    if report["forbidden_jobs_present"]:
        print("\n🔴 禁止的 legacy monitoring job 仍存在")
        for job in report["forbidden_jobs_present"]:
            print(f"  {job}: {report['forbidden_jobs'][job]}")

    if (
        report["missing_expected_targets"]
        or report["down_expected_targets"]
        or report["ambiguous_expected_targets"]
    ):
        print("\n🔴 精確 Target identity/readback 未通過")
        for target_id, target in report["expected_targets"].items():
            if target["state"] != "up":
                print(
                    f"  {target_id}: {target['state']} "
                    f"matches={target['match_count']} {target['description']}"
                )

    stabilization = report.get("stabilization")
    if stabilization and stabilization["attempt"] > 1:
        print("\n⏱️  Prometheus target 穩定化")
        print(
            "  "
            f"{stabilization['status']} after "
            f"{stabilization['attempt']}/{stabilization['attempts']} attempts"
        )

    pct = s["expected_coverage_pct"]
    threshold = COVERAGE_THRESHOLD
    exact_targets_healthy = not (
        report["missing_expected_targets"]
        or report["down_expected_targets"]
        or report["ambiguous_expected_targets"]
    )
    monitoring_contract_healthy = (
        exact_targets_healthy and not report["forbidden_jobs_present"]
    )
    if (
        pct >= threshold
        and not report["real_down_jobs"]
        and monitoring_contract_healthy
    ):
        print(f"\n✅ 監控健康: 覆蓋率 {pct}% >= {threshold}%，無真實問題\n")
    elif pct >= threshold:
        print(f"\n⚠️  覆蓋率達標 ({pct}%)，但有 {s['real_down_jobs']} 個真實 DOWN 需處理\n")
    else:
        print(f"\n❌ 覆蓋率不足: {pct}% < {threshold}%\n")


def main() -> None:
    parser = argparse.ArgumentParser(description="AWOOOI 監控覆蓋率自動發現")
    parser.add_argument("--json", action="store_true", help="輸出 JSON 格式")
    parser.add_argument(
        "--check",
        action="store_true",
        help=f"CI 模式: 覆蓋率 < {COVERAGE_THRESHOLD}% 則 exit 1",
    )
    parser.add_argument(
        "--stabilization-attempts",
        type=int,
        default=_int_env(
            "AWOOOI_MONITORING_TARGET_STABILIZATION_ATTEMPTS",
            DEFAULT_STABILIZATION_ATTEMPTS,
        ),
        help="CI 模式: Prometheus target 狀態重查次數",
    )
    parser.add_argument(
        "--stabilization-sleep-seconds",
        type=float,
        default=_float_env(
            "AWOOOI_MONITORING_TARGET_STABILIZATION_SLEEP_SECONDS",
            DEFAULT_STABILIZATION_SLEEP_SECONDS,
        ),
        help="CI 模式: Prometheus target 重查間隔秒數",
    )
    args = parser.parse_args()

    if args.check:
        report = build_stabilized_report(
            get_prometheus_targets,
            attempts=args.stabilization_attempts,
            sleep_seconds=args.stabilization_sleep_seconds,
        )
    else:
        report = build_report_from_targets(get_prometheus_targets())

    if args.json:
        print(json.dumps(report, ensure_ascii=False, indent=2))
    else:
        print_human_report(report)

    if args.check:
        pct = report["summary"]["expected_coverage_pct"]
        real_down = report["summary"]["real_down_jobs"]
        exact_target_failures = sum(
            report["summary"][key]
            for key in (
                "expected_targets_missing",
                "expected_targets_down",
                "expected_targets_ambiguous",
            )
        )
        forbidden_jobs = report["summary"]["forbidden_jobs_present"]
        if (
            pct < COVERAGE_THRESHOLD
            or real_down > 0
            or exact_target_failures > 0
            or forbidden_jobs > 0
        ):
            sys.exit(1)


if __name__ == "__main__":
    main()