532 lines
20 KiB
Python
532 lines
20 KiB
Python
"""Truthful product-integration readback for the four AI agents.
|
||
|
||
Source wiring and runtime telemetry are reported separately so a class import or
|
||
configured fallback cannot be mistaken for a working production automation loop.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
from typing import Any, Mapping
|
||
|
||
from sqlalchemy import text
|
||
|
||
from services.external_mcp_rag_integration_service import (
|
||
build_external_mcp_rag_integration_readback,
|
||
)
|
||
from services.internal_rag_candidate_canary_service import (
|
||
run_internal_rag_candidate_canary,
|
||
)
|
||
|
||
|
||
POLICY = "runtime_truth_ai_agent_product_integration_v1"
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
|
||
AGENT_SPECS: dict[str, dict[str, Any]] = {
|
||
"hermes": {
|
||
"label": "Hermes",
|
||
"role": "競價情報分析",
|
||
"source_path": "services/hermes_analyst_service.py",
|
||
"source_markers": ["class HermesAnalystService", "def analyze"],
|
||
"scheduler_path": "scheduler.py",
|
||
"scheduler_markers": ["HermesAnalystService", "run_icaim"],
|
||
},
|
||
"nemotron": {
|
||
"label": "NemoTron",
|
||
"role": "受控行動派發",
|
||
"source_path": "services/nemoton_dispatcher_service.py",
|
||
"source_markers": ["class Nemotron", "dispatch"],
|
||
"scheduler_path": "scheduler.py",
|
||
"scheduler_markers": ["NemoTron", "dispatch"],
|
||
},
|
||
"openclaw": {
|
||
"label": "OpenClaw",
|
||
"role": "策略、報表與學習寫回",
|
||
"source_path": "services/openclaw_strategist_service.py",
|
||
"source_markers": ["daily_report", "ai_insights"],
|
||
"scheduler_path": "scheduler.py",
|
||
"scheduler_markers": ["OpenClaw", "daily"],
|
||
},
|
||
"elephant_alpha": {
|
||
"label": "ElephantAlpha",
|
||
"role": "跨 Agent 編排與自癒",
|
||
"source_path": "services/elephant_alpha_autonomous_engine.py",
|
||
"source_markers": ["class ElephantAlphaAutonomousEngine", "action_plans"],
|
||
"scheduler_path": "run_scheduler.py",
|
||
"scheduler_markers": ["ElephantAlpha", "Autonomous engine"],
|
||
},
|
||
}
|
||
|
||
|
||
def _agent_key(value: Any) -> str | None:
|
||
caller = str(value or "").strip().lower().replace("-", "_")
|
||
if "hermes" in caller:
|
||
return "hermes"
|
||
if "nemotron" in caller or "nemoton" in caller or caller.startswith("nim_"):
|
||
return "nemotron"
|
||
if "openclaw" in caller or "open_claw" in caller:
|
||
return "openclaw"
|
||
if "elephant" in caller or caller.startswith("ea_"):
|
||
return "elephant_alpha"
|
||
return None
|
||
|
||
|
||
def _row_dict(row: Any) -> dict[str, Any]:
|
||
mapping = getattr(row, "_mapping", None)
|
||
return dict(mapping) if mapping is not None else dict(row or {})
|
||
|
||
|
||
def _query_rows(sql: str, params: Mapping[str, Any]) -> tuple[list[dict[str, Any]], str | None]:
|
||
session = None
|
||
try:
|
||
from database.manager import get_session
|
||
|
||
session = get_session()
|
||
rows = session.execute(text(sql), dict(params)).fetchall()
|
||
return [_row_dict(row) for row in rows], None
|
||
except (Exception, SystemExit) as exc:
|
||
return [], f"{type(exc).__name__}: {str(exc)[:240]}"
|
||
finally:
|
||
if session is not None:
|
||
session.close()
|
||
|
||
|
||
def _blank_agent_metrics() -> dict[str, dict[str, Any]]:
|
||
return {
|
||
key: {
|
||
"calls": 0,
|
||
"errors": 0,
|
||
"rag_hits": 0,
|
||
"fallbacks": 0,
|
||
"mcp_calls": 0,
|
||
"rag_queries": 0,
|
||
"rag_query_hits": 0,
|
||
"ai_insights": 0,
|
||
"action_plans": 0,
|
||
"executed_action_plans": 0,
|
||
"last_called": None,
|
||
}
|
||
for key in AGENT_SPECS
|
||
}
|
||
|
||
|
||
def _collect_runtime_telemetry(since_at: datetime) -> dict[str, Any]:
|
||
agents = _blank_agent_metrics()
|
||
errors: list[str] = []
|
||
ai_rows, error = _query_rows(
|
||
"""
|
||
SELECT caller,
|
||
COUNT(*) AS calls,
|
||
COUNT(*) FILTER (
|
||
WHERE lower(COALESCE(status, '')) IN
|
||
('error', 'failed', 'failure', 'timeout', 'cancelled')
|
||
OR error IS NOT NULL
|
||
) AS errors,
|
||
COUNT(*) FILTER (WHERE COALESCE(rag_hit, false)) AS rag_hits,
|
||
COUNT(*) FILTER (WHERE fallback_to IS NOT NULL) AS fallbacks,
|
||
MAX(called_at) AS last_called
|
||
FROM ai_calls
|
||
WHERE called_at >= :since_at
|
||
GROUP BY caller
|
||
""",
|
||
{"since_at": since_at},
|
||
)
|
||
if error:
|
||
errors.append(f"ai_calls:{error}")
|
||
unmatched_callers: list[dict[str, Any]] = []
|
||
for row in ai_rows:
|
||
key = _agent_key(row.get("caller"))
|
||
if key is None:
|
||
unmatched_callers.append(
|
||
{"caller": row.get("caller"), "calls": int(row.get("calls") or 0)}
|
||
)
|
||
continue
|
||
metrics = agents[key]
|
||
metrics["calls"] += int(row.get("calls") or 0)
|
||
metrics["errors"] += int(row.get("errors") or 0)
|
||
metrics["rag_hits"] += int(row.get("rag_hits") or 0)
|
||
metrics["fallbacks"] += int(row.get("fallbacks") or 0)
|
||
last_called = row.get("last_called")
|
||
if last_called and (
|
||
not metrics["last_called"] or str(last_called) > str(metrics["last_called"])
|
||
):
|
||
metrics["last_called"] = str(last_called)
|
||
|
||
mcp_rows, error = _query_rows(
|
||
"""
|
||
SELECT caller, COUNT(*) AS calls
|
||
FROM mcp_calls
|
||
WHERE called_at >= :since_at
|
||
GROUP BY caller
|
||
""",
|
||
{"since_at": since_at},
|
||
)
|
||
if error:
|
||
errors.append(f"mcp_calls:{error}")
|
||
all_mcp_calls = sum(int(row.get("calls") or 0) for row in mcp_rows)
|
||
for row in mcp_rows:
|
||
key = _agent_key(row.get("caller"))
|
||
if key:
|
||
agents[key]["mcp_calls"] += int(row.get("calls") or 0)
|
||
|
||
rag_rows, error = _query_rows(
|
||
"""
|
||
SELECT caller,
|
||
COUNT(*) AS queries,
|
||
COALESCE(SUM(hit_count), 0) AS hits
|
||
FROM rag_query_log
|
||
WHERE queried_at >= :since_at
|
||
GROUP BY caller
|
||
""",
|
||
{"since_at": since_at},
|
||
)
|
||
if error:
|
||
errors.append(f"rag_query_log:{error}")
|
||
all_rag_queries = sum(int(row.get("queries") or 0) for row in rag_rows)
|
||
all_rag_hits = sum(int(row.get("hits") or 0) for row in rag_rows)
|
||
for row in rag_rows:
|
||
key = _agent_key(row.get("caller"))
|
||
if key:
|
||
agents[key]["rag_queries"] += int(row.get("queries") or 0)
|
||
agents[key]["rag_query_hits"] += int(row.get("hits") or 0)
|
||
|
||
insight_rows, error = _query_rows(
|
||
"""
|
||
SELECT created_by, COUNT(*) AS count
|
||
FROM ai_insights
|
||
WHERE created_at >= :since_at
|
||
GROUP BY created_by
|
||
""",
|
||
{"since_at": since_at.replace(tzinfo=None)},
|
||
)
|
||
if error:
|
||
errors.append(f"ai_insights:{error}")
|
||
insight_total = 0
|
||
for row in insight_rows:
|
||
count = int(row.get("count") or 0)
|
||
insight_total += count
|
||
key = _agent_key(row.get("created_by"))
|
||
if key:
|
||
agents[key]["ai_insights"] += count
|
||
|
||
plan_rows, error = _query_rows(
|
||
"""
|
||
SELECT created_by,
|
||
COUNT(*) AS count,
|
||
COUNT(*) FILTER (
|
||
WHERE status = 'executed' OR executed_at IS NOT NULL
|
||
) AS executed
|
||
FROM action_plans
|
||
WHERE created_at >= :since_at
|
||
GROUP BY created_by
|
||
""",
|
||
{"since_at": since_at.replace(tzinfo=None)},
|
||
)
|
||
if error:
|
||
errors.append(f"action_plans:{error}")
|
||
action_plan_total = 0
|
||
executed_action_plan_total = 0
|
||
for row in plan_rows:
|
||
count = int(row.get("count") or 0)
|
||
executed = int(row.get("executed") or 0)
|
||
action_plan_total += count
|
||
executed_action_plan_total += executed
|
||
key = _agent_key(row.get("created_by"))
|
||
if key:
|
||
agents[key]["action_plans"] += count
|
||
agents[key]["executed_action_plans"] += executed
|
||
|
||
outcome_rows, error = _query_rows(
|
||
"SELECT COUNT(*) AS count FROM action_outcomes WHERE created_at >= :since_at",
|
||
{"since_at": since_at.replace(tzinfo=None)},
|
||
)
|
||
if error:
|
||
errors.append(f"action_outcomes:{error}")
|
||
action_outcome_total = int(outcome_rows[0].get("count") or 0) if outcome_rows else 0
|
||
|
||
heal_rows, error = _query_rows(
|
||
"""
|
||
SELECT COUNT(*) AS count,
|
||
COUNT(*) FILTER (WHERE result = 'success') AS success
|
||
FROM heal_logs
|
||
WHERE created_at >= :since_at
|
||
""",
|
||
{"since_at": since_at.replace(tzinfo=None)},
|
||
)
|
||
if error:
|
||
errors.append(f"heal_logs:{error}")
|
||
heal_total = int(heal_rows[0].get("count") or 0) if heal_rows else 0
|
||
heal_success = int(heal_rows[0].get("success") or 0) if heal_rows else 0
|
||
|
||
retry_rows, error = _query_rows(
|
||
"""
|
||
SELECT COUNT(DISTINCT incidents.id) AS count
|
||
FROM incidents
|
||
JOIN heal_logs ON heal_logs.incident_id = incidents.id
|
||
WHERE incidents.created_at >= :since_at
|
||
AND incidents.retry_count > 0
|
||
AND incidents.status = 'closed'
|
||
AND heal_logs.result = 'success'
|
||
""",
|
||
{"since_at": since_at.replace(tzinfo=None)},
|
||
)
|
||
if error:
|
||
errors.append(f"verified_retry_or_rollback_incidents:{error}")
|
||
verified_retry_or_rollback_incidents = (
|
||
int(retry_rows[0].get("count") or 0) if retry_rows else 0
|
||
)
|
||
|
||
queue_rows, error = _query_rows(
|
||
"SELECT status, COUNT(*) AS count FROM embedding_retry_queue GROUP BY status",
|
||
{},
|
||
)
|
||
if error:
|
||
errors.append(f"embedding_retry_queue:{error}")
|
||
queue_counts = {
|
||
str(row.get("status") or "unknown"): int(row.get("count") or 0)
|
||
for row in queue_rows
|
||
}
|
||
|
||
return {
|
||
"agents": agents,
|
||
"unmatched_ai_callers": unmatched_callers,
|
||
"totals": {
|
||
"ai_calls": sum(item["calls"] for item in agents.values()),
|
||
"ai_errors": sum(item["errors"] for item in agents.values()),
|
||
"mcp_calls": all_mcp_calls,
|
||
"agent_mapped_mcp_calls": sum(
|
||
item["mcp_calls"] for item in agents.values()
|
||
),
|
||
"rag_queries": all_rag_queries,
|
||
"agent_mapped_rag_queries": sum(
|
||
item["rag_queries"] for item in agents.values()
|
||
),
|
||
"rag_query_hits": all_rag_hits,
|
||
"ai_insights": insight_total,
|
||
"action_plans": action_plan_total,
|
||
"executed_action_plans": executed_action_plan_total,
|
||
"action_outcomes": action_outcome_total,
|
||
"heal_logs": heal_total,
|
||
"heal_success": heal_success,
|
||
"verified_retry_or_rollback_incidents": (
|
||
verified_retry_or_rollback_incidents
|
||
),
|
||
},
|
||
"embedding_retry_queue": queue_counts,
|
||
"read_errors": errors,
|
||
}
|
||
|
||
|
||
def _source_surface(spec: Mapping[str, Any]) -> dict[str, Any]:
|
||
source_path = ROOT / str(spec["source_path"])
|
||
scheduler_path = ROOT / str(spec["scheduler_path"])
|
||
try:
|
||
source_text = source_path.read_text(encoding="utf-8")
|
||
except OSError:
|
||
source_text = ""
|
||
try:
|
||
scheduler_text = scheduler_path.read_text(encoding="utf-8")
|
||
except OSError:
|
||
scheduler_text = ""
|
||
source_markers = {
|
||
marker: marker in source_text for marker in list(spec.get("source_markers") or [])
|
||
}
|
||
scheduler_markers = {
|
||
marker: marker in scheduler_text
|
||
for marker in list(spec.get("scheduler_markers") or [])
|
||
}
|
||
return {
|
||
"source_path": str(spec["source_path"]),
|
||
"source_exists": source_path.exists(),
|
||
"source_markers": source_markers,
|
||
"source_wired": source_path.exists() and all(source_markers.values()),
|
||
"scheduler_path": str(spec["scheduler_path"]),
|
||
"scheduler_exists": scheduler_path.exists(),
|
||
"scheduler_markers": scheduler_markers,
|
||
"scheduler_wired": scheduler_path.exists() and all(scheduler_markers.values()),
|
||
}
|
||
|
||
|
||
def _agent_readback(
|
||
key: str,
|
||
metrics: Mapping[str, Any],
|
||
) -> dict[str, Any]:
|
||
spec = AGENT_SPECS[key]
|
||
source = _source_surface(spec)
|
||
calls = int(metrics.get("calls") or 0)
|
||
errors = int(metrics.get("errors") or 0)
|
||
error_rate = round(errors / calls, 4) if calls else None
|
||
runtime_active = calls > 0
|
||
runtime_healthy = runtime_active and errors / calls <= 0.2
|
||
source_complete = source["source_wired"] and source["scheduler_wired"]
|
||
integration_complete = source_complete and runtime_healthy
|
||
if not source_complete:
|
||
status = "source_incomplete"
|
||
elif not runtime_active:
|
||
status = "runtime_inactive"
|
||
elif not runtime_healthy:
|
||
status = "runtime_degraded"
|
||
else:
|
||
status = "runtime_active"
|
||
return {
|
||
"id": key,
|
||
"label": spec["label"],
|
||
"role": spec["role"],
|
||
"status": status,
|
||
"source": source,
|
||
"runtime": {
|
||
**dict(metrics),
|
||
"active_in_window": runtime_active,
|
||
"error_rate": error_rate,
|
||
"healthy_in_window": runtime_healthy,
|
||
},
|
||
"integration_complete": integration_complete,
|
||
"next_machine_action": (
|
||
"repair_agent_source_or_scheduler_wiring"
|
||
if not source_complete
|
||
else (
|
||
"execute_bounded_agent_runtime_canary"
|
||
if not runtime_active
|
||
else (
|
||
"repair_agent_error_path_and_replay"
|
||
if not runtime_healthy
|
||
else "continue_runtime_closure_verification"
|
||
)
|
||
)
|
||
),
|
||
}
|
||
|
||
|
||
def build_ai_agent_product_integration_readback(
|
||
*,
|
||
window_hours: int = 168,
|
||
) -> dict[str, Any]:
|
||
"""Return source, runtime and product-closure truth for all four agents."""
|
||
window = max(1, min(int(window_hours or 168), 24 * 31))
|
||
now = datetime.now(timezone.utc)
|
||
since_at = now - timedelta(hours=window)
|
||
telemetry = _collect_runtime_telemetry(since_at)
|
||
agents = [
|
||
_agent_readback(key, telemetry["agents"].get(key) or {})
|
||
for key in AGENT_SPECS
|
||
]
|
||
external = build_external_mcp_rag_integration_readback()
|
||
mcp_runtime = dict((external.get("runtime") or {}).get("mcp") or {})
|
||
rag_runtime = dict((external.get("runtime") or {}).get("rag") or {})
|
||
rag_canary = run_internal_rag_candidate_canary(execute=False)
|
||
latest_canary = dict(rag_canary.get("latest_execution") or {})
|
||
totals = telemetry["totals"]
|
||
active_agents = sum(
|
||
1 for item in agents if item["runtime"]["active_in_window"]
|
||
)
|
||
healthy_agents = sum(
|
||
1 for item in agents if item["runtime"]["healthy_in_window"]
|
||
)
|
||
source_agents = sum(
|
||
1
|
||
for item in agents
|
||
if item["source"]["source_wired"] and item["source"]["scheduler_wired"]
|
||
)
|
||
stages = [
|
||
{"stage": "Detect", "passed": totals["ai_calls"] > 0, "evidence": totals["ai_calls"]},
|
||
{"stage": "Normalize", "passed": source_agents == len(agents), "evidence": source_agents},
|
||
{"stage": "Correlate", "passed": active_agents == len(agents), "evidence": active_agents},
|
||
{"stage": "Decide", "passed": totals["action_plans"] > 0, "evidence": totals["action_plans"]},
|
||
{"stage": "Check", "passed": healthy_agents == len(agents), "evidence": healthy_agents},
|
||
{"stage": "Controlled Apply", "passed": totals["executed_action_plans"] > 0, "evidence": totals["executed_action_plans"]},
|
||
{"stage": "Verify", "passed": totals["action_outcomes"] > 0 or totals["heal_success"] > 0, "evidence": totals["action_outcomes"] + totals["heal_success"]},
|
||
{
|
||
"stage": "Retry/Rollback",
|
||
"passed": totals["verified_retry_or_rollback_incidents"] > 0,
|
||
"evidence": totals["verified_retry_or_rollback_incidents"],
|
||
},
|
||
{"stage": "Learn/Writeback", "passed": totals["ai_insights"] > 0 and totals["rag_query_hits"] > 0, "evidence": totals["rag_query_hits"]},
|
||
]
|
||
stage_passed = sum(1 for stage in stages if stage["passed"])
|
||
blockers: list[str] = []
|
||
if source_agents != len(agents):
|
||
blockers.append("agent_source_or_scheduler_wiring_incomplete")
|
||
if active_agents != len(agents):
|
||
blockers.append("not_all_agents_active_in_runtime_window")
|
||
if healthy_agents != len(agents):
|
||
blockers.append("not_all_agents_healthy_in_runtime_window")
|
||
if mcp_runtime.get("enabled") is not True:
|
||
blockers.append("mcp_router_runtime_disabled")
|
||
if rag_runtime.get("enabled") is not True:
|
||
blockers.append("rag_runtime_disabled")
|
||
if totals["mcp_calls"] == 0:
|
||
blockers.append("mcp_runtime_telemetry_empty")
|
||
if totals["rag_queries"] == 0:
|
||
blockers.append("rag_runtime_telemetry_empty")
|
||
if latest_canary.get("canary_passed") is not True:
|
||
blockers.append("internal_rag_candidate_canary_not_proven")
|
||
if stage_passed != len(stages):
|
||
blockers.append("agent_controlled_apply_loop_not_closed")
|
||
if telemetry["read_errors"]:
|
||
blockers.append("runtime_telemetry_read_degraded")
|
||
full_integration = not blockers
|
||
status = "fully_integrated" if full_integration else "partially_integrated"
|
||
|
||
return {
|
||
"success": True,
|
||
"policy": POLICY,
|
||
"generated_at": now.isoformat(),
|
||
"status": status,
|
||
"answer_to_owner": (
|
||
f"四個 AI Agent source/排程 wiring={source_agents}/4；正式環境尚未完整整合："
|
||
f"最近 {window} 小時只有 {active_agents}/4 個 Agent 有實際呼叫，"
|
||
f"完整閉環 {stage_passed}/{len(stages)} 階段，MCP/RAG runtime 與 canary 必須以實證補齊。"
|
||
if not full_integration
|
||
else "四個 AI Agent 已有 source、runtime、MCP/RAG 與受控執行閉環實證。"
|
||
),
|
||
"window": {
|
||
"hours": window,
|
||
"since_at": since_at.isoformat(),
|
||
},
|
||
"completion": {
|
||
"agent_count": len(agents),
|
||
"source_wired_agents": source_agents,
|
||
"runtime_active_agents": active_agents,
|
||
"runtime_healthy_agents": healthy_agents,
|
||
"source_percent": round(source_agents / len(agents) * 100, 1),
|
||
"runtime_active_percent": round(active_agents / len(agents) * 100, 1),
|
||
"closure_stage_passed": stage_passed,
|
||
"closure_stage_total": len(stages),
|
||
"closure_percent": round(stage_passed / len(stages) * 100, 1),
|
||
"full_product_integration": full_integration,
|
||
},
|
||
"agents": agents,
|
||
"closure_stages": stages,
|
||
"runtime_dependencies": {
|
||
"mcp": {
|
||
"enabled": mcp_runtime.get("enabled") is True,
|
||
"telemetry_calls": totals["mcp_calls"],
|
||
},
|
||
"rag": {
|
||
"enabled": rag_runtime.get("enabled") is True,
|
||
"telemetry_queries": totals["rag_queries"],
|
||
"telemetry_hits": totals["rag_query_hits"],
|
||
"latest_candidate_canary": latest_canary,
|
||
},
|
||
"pixelrag": (external.get("runtime") or {}).get("pixelrag") or {},
|
||
},
|
||
"telemetry": telemetry,
|
||
"blockers": blockers,
|
||
"controlled_apply": {
|
||
"database_read": True,
|
||
"database_write": False,
|
||
"network_call": False,
|
||
"model_call": False,
|
||
"secret_read": False,
|
||
},
|
||
"next_machine_action": (
|
||
"execute_internal_rag_candidate_canary_then_activate_shadow_runtime"
|
||
if not full_integration
|
||
else "continue_scheduled_agent_product_integration_verification"
|
||
),
|
||
}
|
||
|
||
|
||
__all__ = ["POLICY", "build_ai_agent_product_integration_readback"]
|