507 lines
20 KiB
Python
507 lines
20 KiB
Python
"""
|
||
Weekly Report Service - Phase 21.3 定期報告
|
||
============================================
|
||
|
||
整合 StatsService + K3sMonitorService，生成週報並推送 Telegram。
|
||
|
||
數據來源:
|
||
- StatsService (告警/AI 效能統計)
|
||
- K3sMonitorService (K3s 健康)
|
||
- Git API (開發活動)
|
||
|
||
符合 leWOOOgo 積木化規範:
|
||
- Service 層封裝所有邏輯
|
||
- 透過 DI 注入依賴
|
||
|
||
@author Claude Code (首席架構師)
|
||
@version 1.0.0
|
||
@date 2026-03-31 (台北時間)
|
||
@see ADR-041-periodic-reporting-architecture.md
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import subprocess
|
||
from datetime import datetime, timedelta
|
||
from pathlib import Path
|
||
from urllib.parse import quote, urlencode
|
||
from urllib.request import Request, urlopen
|
||
from typing import Protocol, runtime_checkable
|
||
|
||
import structlog
|
||
from zoneinfo import ZoneInfo
|
||
|
||
from src.core.config import settings
|
||
from src.services.stats_service import StatsService, get_stats_service
|
||
from src.services.k3s_monitor_service import K3sMonitorService, get_k3s_monitor_service
|
||
from src.services.telegram_gateway import WeeklyReportMessage, get_telegram_gateway
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
# 台北時區
|
||
TZ_TAIPEI = ZoneInfo("Asia/Taipei")
|
||
|
||
|
||
# =============================================================================
|
||
# Protocol (Interface)
|
||
# =============================================================================
|
||
|
||
|
||
@runtime_checkable
|
||
class IWeeklyReportService(Protocol):
|
||
"""
|
||
週報服務介面
|
||
|
||
Phase 21.3: 定義 Protocol 供依賴注入
|
||
"""
|
||
|
||
async def generate_report(self) -> WeeklyReportMessage:
|
||
"""生成週報"""
|
||
...
|
||
|
||
async def send_weekly_report(self) -> bool:
|
||
"""發送週報"""
|
||
...
|
||
|
||
|
||
# =============================================================================
|
||
# Implementation
|
||
# =============================================================================
|
||
|
||
|
||
class WeeklyReportService:
|
||
"""
|
||
週報服務實作
|
||
|
||
整合多個數據來源生成週報
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
stats_service: StatsService | None = None,
|
||
k3s_monitor: K3sMonitorService | None = None,
|
||
):
|
||
self._stats_service = stats_service or get_stats_service()
|
||
self._k3s_monitor = k3s_monitor or get_k3s_monitor_service()
|
||
|
||
def _get_week_range(self) -> tuple[str, datetime, datetime]:
|
||
"""取得本週範圍"""
|
||
now = datetime.now(TZ_TAIPEI)
|
||
# 找到本週一
|
||
monday = now - timedelta(days=now.weekday())
|
||
monday = monday.replace(hour=0, minute=0, second=0, microsecond=0)
|
||
# 本週日
|
||
sunday = monday + timedelta(days=6, hours=23, minutes=59, seconds=59)
|
||
|
||
# ISO 週次
|
||
week_num = now.isocalendar()[1]
|
||
week_range = f"{now.year}-W{week_num:02d}"
|
||
|
||
return week_range, monday, sunday
|
||
|
||
def _get_git_stats(self, since: datetime) -> tuple[int, int, bool]:
|
||
"""取得 Git 統計 (commits, deploys)"""
|
||
git_cwd = "/app"
|
||
if not self._has_local_git_worktree(git_cwd):
|
||
return self._get_gitea_commit_stats(since)
|
||
try:
|
||
# 取得本週 commits 數量
|
||
since_str = since.strftime("%Y-%m-%d")
|
||
result = subprocess.run(
|
||
["git", "log", f"--since={since_str}", "--oneline"],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=10,
|
||
cwd=git_cwd, # K8s 容器內的工作目錄
|
||
)
|
||
if result.returncode != 0:
|
||
logger.warning(
|
||
"git_stats_commits_failed",
|
||
returncode=result.returncode,
|
||
stderr=result.stderr[-300:],
|
||
)
|
||
return self._get_gitea_commit_stats(since)
|
||
commits = len(result.stdout.strip().split("\n")) if result.stdout.strip() else 0
|
||
|
||
# 取得部署次數 (計算含 "deploy" 或 "CD" 的 commits)
|
||
result_deploy = subprocess.run(
|
||
["git", "log", f"--since={since_str}", "--oneline", "--grep=deploy", "--grep=CD", "-i"],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=10,
|
||
cwd=git_cwd,
|
||
)
|
||
if result_deploy.returncode != 0:
|
||
logger.warning(
|
||
"git_stats_deploys_failed",
|
||
returncode=result_deploy.returncode,
|
||
stderr=result_deploy.stderr[-300:],
|
||
)
|
||
return self._get_gitea_commit_stats(since)
|
||
deploys = len(result_deploy.stdout.strip().split("\n")) if result_deploy.stdout.strip() else 0
|
||
|
||
return commits, deploys, True
|
||
except Exception as e:
|
||
logger.warning("git_stats_failed", error=str(e))
|
||
return self._get_gitea_commit_stats(since)
|
||
|
||
def _has_local_git_worktree(self, cwd: str) -> bool:
|
||
"""確認容器內是否真的保留 Git metadata。"""
|
||
return Path(cwd, ".git").exists()
|
||
|
||
def _get_gitea_commit_stats(self, since: datetime) -> tuple[int, int, bool]:
|
||
"""從 Gitea commits API 讀取週報開發活動統計。"""
|
||
owner = quote(str(settings.GITEA_REPO_OWNER), safe="")
|
||
repo = quote(str(settings.GITEA_REPO_NAME), safe="")
|
||
since_utc = since.astimezone(ZoneInfo("UTC")).isoformat().replace("+00:00", "Z")
|
||
header_candidates = self._gitea_read_headers()
|
||
|
||
commits = 0
|
||
deploys = 0
|
||
page = 1
|
||
limit = 50
|
||
max_pages = 1
|
||
try:
|
||
for api_url in self._gitea_api_candidates():
|
||
for headers in header_candidates:
|
||
commits = deploys = 0
|
||
page = 1
|
||
total_commits: int | None = None
|
||
try:
|
||
while page <= max_pages:
|
||
query = urlencode({
|
||
"sha": "main",
|
||
"since": since_utc,
|
||
"page": page,
|
||
"limit": limit,
|
||
})
|
||
url = f"{api_url}/api/v1/repos/{owner}/{repo}/commits?{query}"
|
||
request = Request(url, headers=headers)
|
||
with urlopen(request, timeout=1) as response:
|
||
total_header = response.headers.get("X-Total-Count") or response.headers.get("X-Total")
|
||
if total_header and total_header.isdigit():
|
||
total_commits = int(total_header)
|
||
payload = json.loads(response.read().decode("utf-8"))
|
||
if not isinstance(payload, list):
|
||
logger.warning("gitea_git_stats_unexpected_payload", payload_type=type(payload).__name__)
|
||
break
|
||
commits += len(payload)
|
||
for item in payload:
|
||
commit = item.get("commit") if isinstance(item, dict) else {}
|
||
message = str((commit or {}).get("message") or "")
|
||
subject = message.splitlines()[0].lower()
|
||
if "deploy" in subject or subject.startswith("chore(cd):"):
|
||
deploys += 1
|
||
if len(payload) < limit:
|
||
return total_commits or commits, deploys, True
|
||
page += 1
|
||
if total_commits is not None and total_commits > 0:
|
||
logger.info(
|
||
"gitea_git_stats_sampled",
|
||
total_commits=total_commits,
|
||
sampled_pages=max_pages,
|
||
sampled_deploys=deploys,
|
||
)
|
||
return total_commits, deploys, True
|
||
except Exception as candidate_exc:
|
||
logger.warning(
|
||
"gitea_git_stats_candidate_failed",
|
||
api_url=api_url,
|
||
auth_mode="token" if "Authorization" in headers else "anonymous",
|
||
error=str(candidate_exc),
|
||
)
|
||
continue
|
||
except Exception as exc:
|
||
logger.warning("gitea_git_stats_failed", error=str(exc))
|
||
return 0, 0, False
|
||
|
||
def _gitea_api_candidates(self) -> list[str]:
|
||
"""取得 Git 統計可用的 Gitea read-only API base URLs。"""
|
||
candidates = ["https://gitea.wooo.work", settings.GITEA_API_URL.rstrip("/")]
|
||
deduped: list[str] = []
|
||
for candidate in candidates:
|
||
if candidate and candidate not in deduped:
|
||
deduped.append(candidate)
|
||
return deduped
|
||
|
||
def _gitea_read_headers(self) -> list[dict[str, str]]:
|
||
"""取得 Gitea read-only 統計用 headers，匿名讀優先。"""
|
||
base = {"Accept": "application/json", "User-Agent": "awoooi-weekly-report/1.0"}
|
||
if not settings.GITEA_API_TOKEN:
|
||
return [base]
|
||
token_headers = {**base, "Authorization": f"token {settings.GITEA_API_TOKEN}"}
|
||
return [base, token_headers]
|
||
|
||
async def generate_report(self) -> WeeklyReportMessage:
|
||
"""
|
||
生成週報
|
||
|
||
整合多個數據來源
|
||
"""
|
||
now = datetime.now(TZ_TAIPEI)
|
||
week_range, monday, sunday = self._get_week_range()
|
||
report_date = now.strftime("%Y-%m-%d %H:%M")
|
||
|
||
# 取得統計數據 (7 天)
|
||
stats_source_ok = True
|
||
try:
|
||
incident_summary = await self._stats_service.get_incident_summary(days=7)
|
||
resolution_stats = await self._stats_service.get_resolution_stats(days=7)
|
||
ai_performance = await self._stats_service.get_ai_performance(days=7)
|
||
except Exception as e:
|
||
logger.warning("stats_fetch_failed", error=str(e))
|
||
stats_source_ok = False
|
||
incident_summary = {}
|
||
resolution_stats = {}
|
||
ai_performance = {}
|
||
|
||
# 取得 K3s 狀態
|
||
k3s_source_ok = True
|
||
try:
|
||
k3s_status = await self._k3s_monitor.collect_cluster_status()
|
||
except Exception as e:
|
||
logger.warning("k3s_fetch_failed", error=str(e))
|
||
k3s_source_ok = False
|
||
k3s_status = None
|
||
|
||
# 取得 Git 統計
|
||
commits, deploys, git_source_ok = self._get_git_stats(monday)
|
||
|
||
# 計算指標
|
||
total_incidents = incident_summary.get("total_incidents", 0)
|
||
resolved_rate = incident_summary.get("resolved_rate", 0.0)
|
||
|
||
# 從嚴重度分佈中取得 Critical 數量
|
||
severity_dist = incident_summary.get("severity_distribution", [])
|
||
critical_count = sum(
|
||
s.get("count", 0) for s in severity_dist
|
||
if s.get("severity", "").upper() in ["P0", "CRITICAL"]
|
||
)
|
||
|
||
# AI 效能
|
||
ai_proposals = ai_performance.get("total_proposals", 0)
|
||
ai_executed = ai_performance.get("executed_count", 0)
|
||
ai_success_rate = ai_performance.get("success_rate", 0.0)
|
||
|
||
# 平均回應時間
|
||
avg_response = resolution_stats.get("avg_minutes") or 0.0
|
||
|
||
# K3s 指標
|
||
k3s_uptime = 99.9 # 假設值，實際應從 Prometheus 取得
|
||
pod_restarts = k3s_status.pod_restart_48h if k3s_status else 0
|
||
hpa_events = 0 # 需要從 Prometheus 取得 HPA 事件
|
||
|
||
# 2026-04-07 Claude Code: Sprint 4 F1 — 取得處置分佈
|
||
disp_auto = disp_human = disp_manual = disp_cold = disp_total = 0
|
||
try:
|
||
from src.services.anomaly_counter import get_anomaly_counter
|
||
counter = get_anomaly_counter()
|
||
disp_summary, _ = await counter.get_all_disposition_stats()
|
||
disp_auto = disp_summary.get("auto_repair", 0)
|
||
disp_human = disp_summary.get("human_approved", 0)
|
||
disp_manual = disp_summary.get("manual_resolved", 0)
|
||
disp_cold = disp_summary.get("cold_start_trust", 0)
|
||
disp_total = disp_summary.get("total", 0)
|
||
except Exception as _disp_e:
|
||
logger.warning("weekly_report_disposition_failed", error=str(_disp_e))
|
||
|
||
ai_slo_source_ok = True
|
||
ai_slo_auto_execute_success_rate: float | None = None
|
||
ai_slo_auto_execute_sample_count = 0
|
||
ai_slo_auto_execute_threshold = 0.85
|
||
ai_slo_auto_execute_violated = False
|
||
ai_slo_top_failure = ""
|
||
ai_slo_verifier_coverage_rate: float | None = None
|
||
ai_slo_unverified_auto_count = 0
|
||
try:
|
||
from src.services.adr100_slo_status_service import (
|
||
get_adr100_slo_status_service,
|
||
)
|
||
from src.services.ai_slo_calculator import AiSloCalculator
|
||
|
||
slo_report = await AiSloCalculator(project_id="awoooi").calculate()
|
||
auto_metric = next(
|
||
(metric for metric in slo_report.metrics if metric.name == "auto_execute_success_rate"),
|
||
None,
|
||
)
|
||
if auto_metric is not None:
|
||
ai_slo_auto_execute_success_rate = auto_metric.value
|
||
ai_slo_auto_execute_sample_count = auto_metric.sample_count
|
||
ai_slo_auto_execute_threshold = auto_metric.threshold
|
||
ai_slo_auto_execute_violated = auto_metric.violated
|
||
|
||
diagnostics = slo_report.diagnostics.get("auto_execute_success_rate") or {}
|
||
top_failure = (diagnostics.get("top_failure_groups") or [{}])[0]
|
||
if top_failure:
|
||
ai_slo_top_failure = (
|
||
f"{top_failure.get('alertname') or 'unknown'} / "
|
||
f"{top_failure.get('playbook_id') or 'unknown'} ×"
|
||
f"{int(top_failure.get('count') or 0)}: "
|
||
f"{str(top_failure.get('error_signature') or '')[:90]}"
|
||
)
|
||
|
||
adr100_report = await get_adr100_slo_status_service("awoooi").fetch_report()
|
||
verification = adr100_report.get("verification_coverage") or {}
|
||
ai_slo_verifier_coverage_rate = verification.get("coverage_rate")
|
||
ai_slo_unverified_auto_count = int(verification.get("unverified_auto") or 0)
|
||
except Exception as _slo_e:
|
||
ai_slo_source_ok = False
|
||
logger.warning("weekly_report_ai_slo_failed", error=str(_slo_e))
|
||
|
||
report_source_confidence = 0
|
||
report_source_ok = 0
|
||
report_source_total = 0
|
||
report_source_gap_ids: list[str] = []
|
||
report_asset_state_lines: list[str] = []
|
||
try:
|
||
from src.services.ai_agent_report_source_health import build_ai_agent_report_source_health
|
||
|
||
source_health = await build_ai_agent_report_source_health(days=7)
|
||
source_rollups = source_health.get("rollups") or {}
|
||
report_source_confidence = int(source_rollups.get("confidence_percent") or 0)
|
||
report_source_ok = int(source_rollups.get("source_ok_count") or 0)
|
||
report_source_total = int(source_rollups.get("source_count") or 0)
|
||
report_source_gap_ids = [
|
||
str(source.get("work_item_id"))
|
||
for source in source_health.get("source_health", [])
|
||
if source.get("work_item_id")
|
||
][:5]
|
||
report_asset_state_lines = [
|
||
(
|
||
f"{asset.get('label')}: {asset.get('state')} "
|
||
f"{int(asset.get('done_count') or 0)}/"
|
||
f"{int(asset.get('done_count') or 0) + int(asset.get('blocked_count') or 0)}"
|
||
)
|
||
for asset in source_health.get("automation_assets", [])
|
||
][:5]
|
||
except Exception as _source_e:
|
||
logger.warning("weekly_report_source_health_failed", error=str(_source_e))
|
||
|
||
# 組裝週報
|
||
report = WeeklyReportMessage(
|
||
week_range=week_range,
|
||
report_date=report_date,
|
||
alert_total=total_incidents,
|
||
alert_critical=critical_count,
|
||
alert_resolved=int(total_incidents * resolved_rate / 100) if total_incidents > 0 else 0,
|
||
resolved_rate=resolved_rate,
|
||
ai_proposal_count=ai_proposals,
|
||
ai_executed_count=ai_executed,
|
||
ai_success_rate=ai_success_rate,
|
||
avg_response_minutes=avg_response,
|
||
k3s_uptime_pct=k3s_uptime,
|
||
pod_restart_total=pod_restarts,
|
||
hpa_scale_events=hpa_events,
|
||
commits_count=commits,
|
||
deploy_count=deploys,
|
||
ai_cost_week=0.0, # 需要從 AI 成本追蹤取得
|
||
ai_tokens_week=0, # 需要從 AI 成本追蹤取得
|
||
disposition_auto=disp_auto,
|
||
disposition_human=disp_human,
|
||
disposition_manual=disp_manual,
|
||
disposition_cold_start=disp_cold,
|
||
disposition_total=disp_total,
|
||
stats_source_ok=stats_source_ok,
|
||
k3s_source_ok=k3s_source_ok,
|
||
git_source_ok=git_source_ok,
|
||
cost_source_ok=False,
|
||
ai_slo_source_ok=ai_slo_source_ok,
|
||
ai_slo_auto_execute_success_rate=ai_slo_auto_execute_success_rate,
|
||
ai_slo_auto_execute_sample_count=ai_slo_auto_execute_sample_count,
|
||
ai_slo_auto_execute_threshold=ai_slo_auto_execute_threshold,
|
||
ai_slo_auto_execute_violated=ai_slo_auto_execute_violated,
|
||
ai_slo_top_failure=ai_slo_top_failure,
|
||
ai_slo_verifier_coverage_rate=ai_slo_verifier_coverage_rate,
|
||
ai_slo_unverified_auto_count=ai_slo_unverified_auto_count,
|
||
all_zero_actionable_anomaly=(
|
||
total_incidents == 0
|
||
and ai_proposals == 0
|
||
and ai_executed == 0
|
||
and commits == 0
|
||
and deploys == 0
|
||
and disp_total == 0
|
||
),
|
||
report_source_confidence_percent=report_source_confidence,
|
||
report_source_ok_count=report_source_ok,
|
||
report_source_total_count=report_source_total,
|
||
report_source_gap_ids=report_source_gap_ids,
|
||
report_asset_state_lines=report_asset_state_lines,
|
||
)
|
||
|
||
logger.info(
|
||
"weekly_report_generated",
|
||
week=week_range,
|
||
alerts=total_incidents,
|
||
commits=commits,
|
||
)
|
||
|
||
return report
|
||
|
||
async def send_weekly_report(self) -> bool:
|
||
"""
|
||
發送週報
|
||
|
||
生成週報並推送到 Telegram
|
||
"""
|
||
try:
|
||
# 生成週報
|
||
report = await self.generate_report()
|
||
|
||
# 取得 Telegram Gateway
|
||
gateway = get_telegram_gateway()
|
||
if not gateway._initialized:
|
||
await gateway.initialize()
|
||
|
||
# 發送訊息
|
||
formatted = report.format()
|
||
result = await gateway.send_text(formatted)
|
||
|
||
if result:
|
||
logger.info("weekly_report_sent", week=report.week_range)
|
||
return True
|
||
else:
|
||
logger.error("weekly_report_failed", week=report.week_range)
|
||
return False
|
||
|
||
except Exception as e:
|
||
logger.error("weekly_report_error", error=str(e))
|
||
return False
|
||
|
||
|
||
# =============================================================================
|
||
# Dependency Injection
|
||
# =============================================================================
|
||
|
||
_weekly_report_service: WeeklyReportService | None = None
|
||
|
||
|
||
def get_weekly_report_service() -> WeeklyReportService:
|
||
"""取得 WeeklyReportService 實例 (Singleton)"""
|
||
global _weekly_report_service
|
||
if _weekly_report_service is None:
|
||
_weekly_report_service = WeeklyReportService()
|
||
return _weekly_report_service
|
||
|
||
|
||
# =============================================================================
|
||
# CLI Entry Point (for CronJob)
|
||
# =============================================================================
|
||
|
||
|
||
async def main():
|
||
"""
|
||
CLI 入口點
|
||
|
||
供 K8s CronJob 調用:
|
||
python -m src.services.weekly_report_service
|
||
"""
|
||
service = get_weekly_report_service()
|
||
success = await service.send_weekly_report()
|
||
return 0 if success else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import sys
|
||
exit_code = asyncio.run(main())
|
||
sys.exit(exit_code)
|