"""Truthful product-integration readback for the four AI agents. Source wiring and runtime telemetry are reported separately so a class import or configured fallback cannot be mistaken for a working production automation loop. """ from __future__ import annotations from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Mapping from sqlalchemy import text from services.external_mcp_rag_integration_service import ( build_external_mcp_rag_integration_readback, ) from services.internal_rag_candidate_canary_service import ( run_internal_rag_candidate_canary, ) POLICY = "runtime_truth_ai_agent_product_integration_v1" ROOT = Path(__file__).resolve().parent.parent AGENT_SPECS: dict[str, dict[str, Any]] = { "hermes": { "label": "Hermes", "role": "競價情報分析", "source_path": "services/hermes_analyst_service.py", "source_markers": ["class HermesAnalystService", "def analyze"], "scheduler_path": "scheduler.py", "scheduler_markers": ["HermesAnalystService", "run_icaim"], }, "nemotron": { "label": "NemoTron", "role": "受控行動派發", "source_path": "services/nemoton_dispatcher_service.py", "source_markers": ["class Nemotron", "dispatch"], "scheduler_path": "scheduler.py", "scheduler_markers": ["NemoTron", "dispatch"], }, "openclaw": { "label": "OpenClaw", "role": "策略、報表與學習寫回", "source_path": "services/openclaw_strategist_service.py", "source_markers": ["daily_report", "ai_insights"], "scheduler_path": "scheduler.py", "scheduler_markers": ["OpenClaw", "daily"], }, "elephant_alpha": { "label": "ElephantAlpha", "role": "跨 Agent 編排與自癒", "source_path": "services/elephant_alpha_autonomous_engine.py", "source_markers": ["class ElephantAlphaAutonomousEngine", "action_plans"], "scheduler_path": "run_scheduler.py", "scheduler_markers": ["ElephantAlpha", "Autonomous engine"], }, } def _agent_key(value: Any) -> str | None: caller = str(value or "").strip().lower().replace("-", "_") if "hermes" in caller: return "hermes" if "nemotron" in caller or "nemoton" in caller or caller.startswith("nim_"): return "nemotron" if "openclaw" in caller or "open_claw" in caller: return "openclaw" if "elephant" in caller or caller.startswith("ea_"): return "elephant_alpha" return None def _row_dict(row: Any) -> dict[str, Any]: mapping = getattr(row, "_mapping", None) return dict(mapping) if mapping is not None else dict(row or {}) def _query_rows(sql: str, params: Mapping[str, Any]) -> tuple[list[dict[str, Any]], str | None]: session = None try: from database.manager import get_session session = get_session() rows = session.execute(text(sql), dict(params)).fetchall() return [_row_dict(row) for row in rows], None except (Exception, SystemExit) as exc: return [], f"{type(exc).__name__}: {str(exc)[:240]}" finally: if session is not None: session.close() def _blank_agent_metrics() -> dict[str, dict[str, Any]]: return { key: { "calls": 0, "errors": 0, "rag_hits": 0, "fallbacks": 0, "mcp_calls": 0, "rag_queries": 0, "rag_query_hits": 0, "ai_insights": 0, "action_plans": 0, "executed_action_plans": 0, "last_called": None, } for key in AGENT_SPECS } def _collect_runtime_telemetry(since_at: datetime) -> dict[str, Any]: agents = _blank_agent_metrics() errors: list[str] = [] ai_rows, error = _query_rows( """ SELECT caller, COUNT(*) AS calls, COUNT(*) FILTER ( WHERE lower(COALESCE(status, '')) IN ('error', 'failed', 'failure', 'timeout', 'cancelled') OR error IS NOT NULL ) AS errors, COUNT(*) FILTER (WHERE COALESCE(rag_hit, false)) AS rag_hits, COUNT(*) FILTER (WHERE fallback_to IS NOT NULL) AS fallbacks, MAX(called_at) AS last_called FROM ai_calls WHERE called_at >= :since_at GROUP BY caller """, {"since_at": since_at}, ) if error: errors.append(f"ai_calls:{error}") unmatched_callers: list[dict[str, Any]] = [] for row in ai_rows: key = _agent_key(row.get("caller")) if key is None: unmatched_callers.append( {"caller": row.get("caller"), "calls": int(row.get("calls") or 0)} ) continue metrics = agents[key] metrics["calls"] += int(row.get("calls") or 0) metrics["errors"] += int(row.get("errors") or 0) metrics["rag_hits"] += int(row.get("rag_hits") or 0) metrics["fallbacks"] += int(row.get("fallbacks") or 0) last_called = row.get("last_called") if last_called and ( not metrics["last_called"] or str(last_called) > str(metrics["last_called"]) ): metrics["last_called"] = str(last_called) mcp_rows, error = _query_rows( """ SELECT caller, COUNT(*) AS calls FROM mcp_calls WHERE called_at >= :since_at GROUP BY caller """, {"since_at": since_at}, ) if error: errors.append(f"mcp_calls:{error}") all_mcp_calls = sum(int(row.get("calls") or 0) for row in mcp_rows) for row in mcp_rows: key = _agent_key(row.get("caller")) if key: agents[key]["mcp_calls"] += int(row.get("calls") or 0) rag_rows, error = _query_rows( """ SELECT caller, COUNT(*) AS queries, COALESCE(SUM(hit_count), 0) AS hits FROM rag_query_log WHERE queried_at >= :since_at GROUP BY caller """, {"since_at": since_at}, ) if error: errors.append(f"rag_query_log:{error}") all_rag_queries = sum(int(row.get("queries") or 0) for row in rag_rows) all_rag_hits = sum(int(row.get("hits") or 0) for row in rag_rows) for row in rag_rows: key = _agent_key(row.get("caller")) if key: agents[key]["rag_queries"] += int(row.get("queries") or 0) agents[key]["rag_query_hits"] += int(row.get("hits") or 0) insight_rows, error = _query_rows( """ SELECT created_by, COUNT(*) AS count FROM ai_insights WHERE created_at >= :since_at GROUP BY created_by """, {"since_at": since_at.replace(tzinfo=None)}, ) if error: errors.append(f"ai_insights:{error}") insight_total = 0 for row in insight_rows: count = int(row.get("count") or 0) insight_total += count key = _agent_key(row.get("created_by")) if key: agents[key]["ai_insights"] += count plan_rows, error = _query_rows( """ SELECT created_by, COUNT(*) AS count, COUNT(*) FILTER ( WHERE status = 'executed' OR executed_at IS NOT NULL ) AS executed FROM action_plans WHERE created_at >= :since_at GROUP BY created_by """, {"since_at": since_at.replace(tzinfo=None)}, ) if error: errors.append(f"action_plans:{error}") action_plan_total = 0 executed_action_plan_total = 0 for row in plan_rows: count = int(row.get("count") or 0) executed = int(row.get("executed") or 0) action_plan_total += count executed_action_plan_total += executed key = _agent_key(row.get("created_by")) if key: agents[key]["action_plans"] += count agents[key]["executed_action_plans"] += executed outcome_rows, error = _query_rows( "SELECT COUNT(*) AS count FROM action_outcomes WHERE created_at >= :since_at", {"since_at": since_at.replace(tzinfo=None)}, ) if error: errors.append(f"action_outcomes:{error}") action_outcome_total = int(outcome_rows[0].get("count") or 0) if outcome_rows else 0 heal_rows, error = _query_rows( """ SELECT COUNT(*) AS count, COUNT(*) FILTER (WHERE result = 'success') AS success FROM heal_logs WHERE created_at >= :since_at """, {"since_at": since_at.replace(tzinfo=None)}, ) if error: errors.append(f"heal_logs:{error}") heal_total = int(heal_rows[0].get("count") or 0) if heal_rows else 0 heal_success = int(heal_rows[0].get("success") or 0) if heal_rows else 0 retry_rows, error = _query_rows( """ SELECT COUNT(DISTINCT incidents.id) AS count FROM incidents JOIN heal_logs ON heal_logs.incident_id = incidents.id WHERE incidents.created_at >= :since_at AND incidents.retry_count > 0 AND incidents.status = 'closed' AND heal_logs.result = 'success' """, {"since_at": since_at.replace(tzinfo=None)}, ) if error: errors.append(f"verified_retry_or_rollback_incidents:{error}") verified_retry_or_rollback_incidents = ( int(retry_rows[0].get("count") or 0) if retry_rows else 0 ) queue_rows, error = _query_rows( "SELECT status, COUNT(*) AS count FROM embedding_retry_queue GROUP BY status", {}, ) if error: errors.append(f"embedding_retry_queue:{error}") queue_counts = { str(row.get("status") or "unknown"): int(row.get("count") or 0) for row in queue_rows } return { "agents": agents, "unmatched_ai_callers": unmatched_callers, "totals": { "ai_calls": sum(item["calls"] for item in agents.values()), "ai_errors": sum(item["errors"] for item in agents.values()), "mcp_calls": all_mcp_calls, "agent_mapped_mcp_calls": sum( item["mcp_calls"] for item in agents.values() ), "rag_queries": all_rag_queries, "agent_mapped_rag_queries": sum( item["rag_queries"] for item in agents.values() ), "rag_query_hits": all_rag_hits, "ai_insights": insight_total, "action_plans": action_plan_total, "executed_action_plans": executed_action_plan_total, "action_outcomes": action_outcome_total, "heal_logs": heal_total, "heal_success": heal_success, "verified_retry_or_rollback_incidents": ( verified_retry_or_rollback_incidents ), }, "embedding_retry_queue": queue_counts, "read_errors": errors, } def _source_surface(spec: Mapping[str, Any]) -> dict[str, Any]: source_path = ROOT / str(spec["source_path"]) scheduler_path = ROOT / str(spec["scheduler_path"]) try: source_text = source_path.read_text(encoding="utf-8") except OSError: source_text = "" try: scheduler_text = scheduler_path.read_text(encoding="utf-8") except OSError: scheduler_text = "" source_markers = { marker: marker in source_text for marker in list(spec.get("source_markers") or []) } scheduler_markers = { marker: marker in scheduler_text for marker in list(spec.get("scheduler_markers") or []) } return { "source_path": str(spec["source_path"]), "source_exists": source_path.exists(), "source_markers": source_markers, "source_wired": source_path.exists() and all(source_markers.values()), "scheduler_path": str(spec["scheduler_path"]), "scheduler_exists": scheduler_path.exists(), "scheduler_markers": scheduler_markers, "scheduler_wired": scheduler_path.exists() and all(scheduler_markers.values()), } def _agent_readback( key: str, metrics: Mapping[str, Any], ) -> dict[str, Any]: spec = AGENT_SPECS[key] source = _source_surface(spec) calls = int(metrics.get("calls") or 0) errors = int(metrics.get("errors") or 0) error_rate = round(errors / calls, 4) if calls else None runtime_active = calls > 0 runtime_healthy = runtime_active and errors / calls <= 0.2 source_complete = source["source_wired"] and source["scheduler_wired"] integration_complete = source_complete and runtime_healthy if not source_complete: status = "source_incomplete" elif not runtime_active: status = "runtime_inactive" elif not runtime_healthy: status = "runtime_degraded" else: status = "runtime_active" return { "id": key, "label": spec["label"], "role": spec["role"], "status": status, "source": source, "runtime": { **dict(metrics), "active_in_window": runtime_active, "error_rate": error_rate, "healthy_in_window": runtime_healthy, }, "integration_complete": integration_complete, "next_machine_action": ( "repair_agent_source_or_scheduler_wiring" if not source_complete else ( "execute_bounded_agent_runtime_canary" if not runtime_active else ( "repair_agent_error_path_and_replay" if not runtime_healthy else "continue_runtime_closure_verification" ) ) ), } def build_ai_agent_product_integration_readback( *, window_hours: int = 168, ) -> dict[str, Any]: """Return source, runtime and product-closure truth for all four agents.""" window = max(1, min(int(window_hours or 168), 24 * 31)) now = datetime.now(timezone.utc) since_at = now - timedelta(hours=window) telemetry = _collect_runtime_telemetry(since_at) agents = [ _agent_readback(key, telemetry["agents"].get(key) or {}) for key in AGENT_SPECS ] external = build_external_mcp_rag_integration_readback() mcp_runtime = dict((external.get("runtime") or {}).get("mcp") or {}) rag_runtime = dict((external.get("runtime") or {}).get("rag") or {}) rag_canary = run_internal_rag_candidate_canary(execute=False) latest_canary = dict(rag_canary.get("latest_execution") or {}) totals = telemetry["totals"] active_agents = sum( 1 for item in agents if item["runtime"]["active_in_window"] ) healthy_agents = sum( 1 for item in agents if item["runtime"]["healthy_in_window"] ) source_agents = sum( 1 for item in agents if item["source"]["source_wired"] and item["source"]["scheduler_wired"] ) stages = [ {"stage": "Detect", "passed": totals["ai_calls"] > 0, "evidence": totals["ai_calls"]}, {"stage": "Normalize", "passed": source_agents == len(agents), "evidence": source_agents}, {"stage": "Correlate", "passed": active_agents == len(agents), "evidence": active_agents}, {"stage": "Decide", "passed": totals["action_plans"] > 0, "evidence": totals["action_plans"]}, {"stage": "Check", "passed": healthy_agents == len(agents), "evidence": healthy_agents}, {"stage": "Controlled Apply", "passed": totals["executed_action_plans"] > 0, "evidence": totals["executed_action_plans"]}, {"stage": "Verify", "passed": totals["action_outcomes"] > 0 or totals["heal_success"] > 0, "evidence": totals["action_outcomes"] + totals["heal_success"]}, { "stage": "Retry/Rollback", "passed": totals["verified_retry_or_rollback_incidents"] > 0, "evidence": totals["verified_retry_or_rollback_incidents"], }, {"stage": "Learn/Writeback", "passed": totals["ai_insights"] > 0 and totals["rag_query_hits"] > 0, "evidence": totals["rag_query_hits"]}, ] stage_passed = sum(1 for stage in stages if stage["passed"]) blockers: list[str] = [] if source_agents != len(agents): blockers.append("agent_source_or_scheduler_wiring_incomplete") if active_agents != len(agents): blockers.append("not_all_agents_active_in_runtime_window") if healthy_agents != len(agents): blockers.append("not_all_agents_healthy_in_runtime_window") if mcp_runtime.get("enabled") is not True: blockers.append("mcp_router_runtime_disabled") if rag_runtime.get("enabled") is not True: blockers.append("rag_runtime_disabled") if totals["mcp_calls"] == 0: blockers.append("mcp_runtime_telemetry_empty") if totals["rag_queries"] == 0: blockers.append("rag_runtime_telemetry_empty") if latest_canary.get("canary_passed") is not True: blockers.append("internal_rag_candidate_canary_not_proven") if stage_passed != len(stages): blockers.append("agent_controlled_apply_loop_not_closed") if telemetry["read_errors"]: blockers.append("runtime_telemetry_read_degraded") full_integration = not blockers status = "fully_integrated" if full_integration else "partially_integrated" return { "success": True, "policy": POLICY, "generated_at": now.isoformat(), "status": status, "answer_to_owner": ( f"四個 AI Agent source/排程 wiring={source_agents}/4；正式環境尚未完整整合：" f"最近 {window} 小時只有 {active_agents}/4 個 Agent 有實際呼叫，" f"完整閉環 {stage_passed}/{len(stages)} 階段，MCP/RAG runtime 與 canary 必須以實證補齊。" if not full_integration else "四個 AI Agent 已有 source、runtime、MCP/RAG 與受控執行閉環實證。" ), "window": { "hours": window, "since_at": since_at.isoformat(), }, "completion": { "agent_count": len(agents), "source_wired_agents": source_agents, "runtime_active_agents": active_agents, "runtime_healthy_agents": healthy_agents, "source_percent": round(source_agents / len(agents) * 100, 1), "runtime_active_percent": round(active_agents / len(agents) * 100, 1), "closure_stage_passed": stage_passed, "closure_stage_total": len(stages), "closure_percent": round(stage_passed / len(stages) * 100, 1), "full_product_integration": full_integration, }, "agents": agents, "closure_stages": stages, "runtime_dependencies": { "mcp": { "enabled": mcp_runtime.get("enabled") is True, "telemetry_calls": totals["mcp_calls"], }, "rag": { "enabled": rag_runtime.get("enabled") is True, "telemetry_queries": totals["rag_queries"], "telemetry_hits": totals["rag_query_hits"], "latest_candidate_canary": latest_canary, }, "pixelrag": (external.get("runtime") or {}).get("pixelrag") or {}, }, "telemetry": telemetry, "blockers": blockers, "controlled_apply": { "database_read": True, "database_write": False, "network_call": False, "model_call": False, "secret_read": False, }, "next_machine_action": ( "execute_internal_rag_candidate_canary_then_activate_shadow_runtime" if not full_integration else "continue_scheduled_agent_product_integration_verification" ), } __all__ = ["POLICY", "build_ai_agent_product_integration_readback"]