1880 lines
83 KiB
Python
1880 lines
83 KiB
Python
"""Smoke checks for the four-agent AI automation control plane.
|
||
|
||
The checks are read-only and intentionally avoid outbound network calls. They
|
||
are meant for a fast dashboard/API sanity check, not for deep production probes.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import json
|
||
import threading
|
||
from datetime import datetime, timedelta, timezone
|
||
from html import escape
|
||
from typing import Any, Dict, List
|
||
|
||
from sqlalchemy import text
|
||
|
||
from config import SYSTEM_VERSION
|
||
from database.manager import get_session
|
||
|
||
|
||
STATUS_RANK = {"ok": 0, "warning": 1, "critical": 2}
|
||
SCHEDULED_AUTOMATION_HEALTH_SUMMARY_POLICY = "read_only_ai_automation_scheduled_health_summary"
|
||
_HISTORY_PATH = os.getenv(
|
||
"MOMO_AI_AUTOMATION_SMOKE_HISTORY",
|
||
os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "ai_automation_smoke_history.jsonl"),
|
||
)
|
||
_HISTORY_LIMIT = int(os.getenv("MOMO_AI_AUTOMATION_SMOKE_HISTORY_LIMIT", "200"))
|
||
_HISTORY_LOCK = threading.Lock()
|
||
_PCHOME_DRIFT_MONITOR_DB_STATEMENT_TIMEOUT_MS = int(
|
||
os.getenv("MOMO_PCHOME_DRIFT_MONITOR_DB_STATEMENT_TIMEOUT_MS", "5000")
|
||
)
|
||
_PCHOME_AUTO_POLICY_GUARD_LIMIT = int(
|
||
os.getenv("MOMO_PCHOME_AUTO_POLICY_GUARD_LIMIT", "20")
|
||
)
|
||
_PCHOME_AUTO_POLICY_GUARD_BATCH_SIZE = int(
|
||
os.getenv("MOMO_PCHOME_AUTO_POLICY_GUARD_BATCH_SIZE", "12")
|
||
)
|
||
_PCHOME_AUTO_POLICY_GUARD_TIMEOUT_SECONDS = int(
|
||
os.getenv("MOMO_PCHOME_AUTO_POLICY_GUARD_TIMEOUT_SECONDS", "5")
|
||
)
|
||
_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_BATCH_SIZE = int(
|
||
os.getenv(
|
||
"MOMO_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_BATCH_SIZE",
|
||
str(_PCHOME_AUTO_POLICY_GUARD_BATCH_SIZE),
|
||
)
|
||
)
|
||
_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_TIMEOUT_SECONDS = int(
|
||
os.getenv(
|
||
"MOMO_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_TIMEOUT_SECONDS",
|
||
str(_PCHOME_AUTO_POLICY_GUARD_TIMEOUT_SECONDS),
|
||
)
|
||
)
|
||
|
||
|
||
def _check(name: str, status: str, summary: str, details: Dict[str, Any] | None = None) -> Dict[str, Any]:
|
||
return {
|
||
"name": name,
|
||
"status": status,
|
||
"summary": summary,
|
||
"details": details or {},
|
||
}
|
||
|
||
|
||
def _count_jsonl_lines(path: str) -> int:
|
||
try:
|
||
with open(path, "r", encoding="utf-8") as fh:
|
||
return sum(1 for line in fh if line.strip())
|
||
except FileNotFoundError:
|
||
return 0
|
||
|
||
|
||
def _compact_history_record(result: Dict[str, Any]) -> Dict[str, Any]:
|
||
return {
|
||
"generated_at": result.get("generated_at"),
|
||
"status": result.get("status", "critical"),
|
||
"summary": result.get("summary", {}),
|
||
"checks": [
|
||
{
|
||
"name": item.get("name"),
|
||
"status": item.get("status"),
|
||
"summary": item.get("summary"),
|
||
}
|
||
for item in result.get("checks", [])
|
||
],
|
||
}
|
||
|
||
|
||
def _read_history_lines() -> List[str]:
|
||
with _HISTORY_LOCK:
|
||
with open(_HISTORY_PATH, "r", encoding="utf-8") as fh:
|
||
return fh.readlines()
|
||
|
||
|
||
def _load_history(limit: int = 20) -> List[Dict[str, Any]]:
|
||
if limit <= 0:
|
||
return []
|
||
try:
|
||
lines = _read_history_lines()
|
||
except FileNotFoundError:
|
||
return []
|
||
|
||
records = []
|
||
for line in lines[-limit:]:
|
||
try:
|
||
records.append(json.loads(line))
|
||
except json.JSONDecodeError:
|
||
continue
|
||
return records
|
||
|
||
|
||
def _append_history(result: Dict[str, Any]) -> None:
|
||
record = _compact_history_record(result)
|
||
os.makedirs(os.path.dirname(_HISTORY_PATH), exist_ok=True)
|
||
with _HISTORY_LOCK:
|
||
existing = []
|
||
try:
|
||
with open(_HISTORY_PATH, "r", encoding="utf-8") as fh:
|
||
existing = fh.readlines()
|
||
except FileNotFoundError:
|
||
pass
|
||
|
||
existing.append(json.dumps(record, ensure_ascii=False, default=str) + "\n")
|
||
if len(existing) > _HISTORY_LIMIT:
|
||
existing = existing[-_HISTORY_LIMIT:]
|
||
|
||
with open(_HISTORY_PATH, "w", encoding="utf-8") as fh:
|
||
fh.writelines(existing)
|
||
|
||
|
||
def _history_summary(records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||
counts = {"ok": 0, "warning": 0, "critical": 0}
|
||
for record in records:
|
||
status = record.get("status", "critical")
|
||
if status in counts:
|
||
counts[status] += 1
|
||
return {
|
||
"counts": counts,
|
||
"recent": records,
|
||
"latest": records[-1] if records else None,
|
||
"daily": _daily_summary(records),
|
||
}
|
||
|
||
|
||
def _parse_history_timestamp(value: Any) -> datetime | None:
|
||
if not value:
|
||
return None
|
||
text_value = str(value).strip()
|
||
if not text_value:
|
||
return None
|
||
try:
|
||
return datetime.fromisoformat(text_value.replace("Z", "+00:00"))
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def _history_freshness_family(
|
||
latest: Dict[str, Any] | None,
|
||
*,
|
||
max_age_hours: int,
|
||
) -> Dict[str, Any]:
|
||
if not latest:
|
||
return {
|
||
"key": "smoke_history_freshness",
|
||
"label": "Smoke history freshness",
|
||
"status": "warning",
|
||
"summary": "No scheduled smoke history is available yet.",
|
||
"next_machine_action": "run_ai_automation_smoke_once",
|
||
"details": {"history_record_count": 0, "writes_database": False},
|
||
}
|
||
generated_at = _parse_history_timestamp(latest.get("generated_at"))
|
||
if generated_at is None:
|
||
return {
|
||
"key": "smoke_history_freshness",
|
||
"label": "Smoke history freshness",
|
||
"status": "warning",
|
||
"summary": "Latest scheduled smoke history timestamp cannot be parsed.",
|
||
"next_machine_action": "refresh_smoke_history_timestamp",
|
||
"details": {
|
||
"generated_at": latest.get("generated_at"),
|
||
"writes_database": False,
|
||
},
|
||
}
|
||
|
||
now = datetime.now(generated_at.tzinfo) if generated_at.tzinfo else datetime.now()
|
||
age_hours = max((now - generated_at).total_seconds() / 3600, 0)
|
||
fresh = age_hours <= max_age_hours
|
||
return {
|
||
"key": "smoke_history_freshness",
|
||
"label": "Smoke history freshness",
|
||
"status": "ok" if fresh else "warning",
|
||
"summary": (
|
||
f"Latest smoke history is {age_hours:.1f}h old."
|
||
if fresh
|
||
else f"Latest smoke history is stale at {age_hours:.1f}h old."
|
||
),
|
||
"next_machine_action": "keep_scheduled_smoke_cadence" if fresh else "run_ai_automation_smoke_once",
|
||
"details": {
|
||
"generated_at": latest.get("generated_at"),
|
||
"age_hours": round(age_hours, 2),
|
||
"max_age_hours": max_age_hours,
|
||
"writes_database": False,
|
||
},
|
||
}
|
||
|
||
|
||
def _find_check(result: Dict[str, Any], name: str) -> Dict[str, Any]:
|
||
for item in result.get("checks", []) or []:
|
||
if item.get("name") == name:
|
||
return item
|
||
return {}
|
||
|
||
|
||
def build_scheduled_automation_health_summary(
|
||
*,
|
||
history_limit: int = 200,
|
||
include_current_smoke: bool = False,
|
||
current_smoke: Dict[str, Any] | None = None,
|
||
freshness_max_age_hours: int = 26,
|
||
) -> Dict[str, Any]:
|
||
"""Build a machine-readable scheduled health summary without sending notifications."""
|
||
records = _load_history(limit=history_limit)
|
||
history = _history_summary(records)
|
||
latest_history = history.get("latest") or {}
|
||
current = current_smoke
|
||
if current is None and include_current_smoke:
|
||
current = collect_ai_automation_smoke(record_history=False, history_limit=history_limit)
|
||
source_result = current or latest_history or {}
|
||
source_kind = "current_smoke" if current else "latest_history"
|
||
pchome_drift = _find_check(source_result, "PChome 受控落地 drift monitor")
|
||
pchome_details = pchome_drift.get("details") or {}
|
||
auto_policy_guard = _find_check(source_result, "PChome auto-policy authorization guard")
|
||
auto_policy_details = auto_policy_guard.get("details") or {}
|
||
if not auto_policy_guard or not auto_policy_details:
|
||
auto_policy_guard = _pchome_auto_policy_authorization_guard_check()
|
||
auto_policy_details = auto_policy_guard.get("details") or {}
|
||
auto_policy_preflight = _find_check(source_result, "PChome auto-policy decision preflight")
|
||
auto_policy_preflight_details = auto_policy_preflight.get("details") or {}
|
||
if not auto_policy_preflight or not auto_policy_preflight_details:
|
||
auto_policy_preflight = _pchome_auto_policy_decision_preflight_check()
|
||
auto_policy_preflight_details = auto_policy_preflight.get("details") or {}
|
||
auto_policy_closeout = _find_check(source_result, "PChome auto-policy decision closeout")
|
||
auto_policy_closeout_details = auto_policy_closeout.get("details") or {}
|
||
if not auto_policy_closeout or not auto_policy_closeout_details:
|
||
auto_policy_closeout = _pchome_auto_policy_decision_closeout_check()
|
||
auto_policy_closeout_details = auto_policy_closeout.get("details") or {}
|
||
surface_readback = _find_check(source_result, "AI surface HTML readback")
|
||
surface_details = surface_readback.get("details") or {}
|
||
sitewide_readback = _find_check(source_result, "Sitewide UI/UX Agent readback")
|
||
sitewide_details = sitewide_readback.get("details") or {}
|
||
visual_qa_readback = _find_check(source_result, "Sitewide visual QA readback")
|
||
visual_qa_details = visual_qa_readback.get("details") or {}
|
||
if not visual_qa_readback or not visual_qa_details:
|
||
visual_qa_readback = _latest_sitewide_visual_qa_readback_check()
|
||
visual_qa_details = visual_qa_readback.get("details") or {}
|
||
smoke_status = source_result.get("status") or ("warning" if latest_history else "warning")
|
||
freshness_family = _history_freshness_family(
|
||
latest_history,
|
||
max_age_hours=max(1, int(freshness_max_age_hours or 26)),
|
||
)
|
||
daily_rows = history.get("daily") or []
|
||
history_count = len(records)
|
||
families = [
|
||
{
|
||
"key": "ai_automation_smoke",
|
||
"label": "AI automation smoke",
|
||
"status": smoke_status,
|
||
"summary": (
|
||
f"Smoke source={source_kind}; "
|
||
f"checks={(source_result.get('summary') or {}).get('total', 0)}"
|
||
),
|
||
"next_machine_action": (
|
||
"keep_monitoring"
|
||
if smoke_status == "ok"
|
||
else "inspect_warning_or_critical_checks"
|
||
),
|
||
"details": {
|
||
"source_kind": source_kind,
|
||
"generated_at": source_result.get("generated_at"),
|
||
"summary": source_result.get("summary") or {},
|
||
"writes_database": False,
|
||
},
|
||
},
|
||
{
|
||
"key": "pchome_controlled_apply_drift_monitor",
|
||
"label": "PChome controlled apply drift monitor",
|
||
"status": pchome_drift.get("status") or "warning",
|
||
"summary": pchome_drift.get("summary") or "PChome drift monitor has no latest check.",
|
||
"next_machine_action": (
|
||
"keep_monitoring_drift"
|
||
if (pchome_drift.get("status") == "ok" and int(pchome_details.get("drift_count") or 0) == 0)
|
||
else "run_pchome_drift_verifier_and_recovery_package"
|
||
),
|
||
"details": {
|
||
"result": pchome_details.get("result"),
|
||
"source_receipt_replay_result": pchome_details.get("source_receipt_replay_result"),
|
||
"selector_count": int(pchome_details.get("selector_count") or 0),
|
||
"readback_pass_count": int(pchome_details.get("readback_pass_count") or 0),
|
||
"drift_count": int(pchome_details.get("drift_count") or 0),
|
||
"writes_database": False,
|
||
},
|
||
},
|
||
{
|
||
"key": "pchome_auto_policy_authorization_guard",
|
||
"label": "PChome auto-policy authorization guard",
|
||
"status": auto_policy_guard.get("status") or "warning",
|
||
"summary": (
|
||
auto_policy_guard.get("summary")
|
||
or "PChome auto-policy authorization guard has no latest check."
|
||
),
|
||
"next_machine_action": auto_policy_details.get("next_machine_action")
|
||
or (
|
||
"continue_pchome_auto_policy_guard_monitoring"
|
||
if auto_policy_guard.get("status") == "ok"
|
||
else "refresh_pchome_auto_policy_authorization_guard"
|
||
),
|
||
"details": {
|
||
"policy": auto_policy_details.get("policy"),
|
||
"result": auto_policy_details.get("result"),
|
||
"guard_ready_count": int(auto_policy_details.get("guard_ready_count") or 0),
|
||
"lane_guard_check_count": int(auto_policy_details.get("lane_guard_check_count") or 0),
|
||
"lane_guard_pass_count": int(auto_policy_details.get("lane_guard_pass_count") or 0),
|
||
"lane_guard_waiting_count": int(auto_policy_details.get("lane_guard_waiting_count") or 0),
|
||
"authorization_request_closeout_ready_count": int(
|
||
auto_policy_details.get("authorization_request_closeout_ready_count") or 0
|
||
),
|
||
"exact_request_payload_field_count": int(
|
||
auto_policy_details.get("exact_request_payload_field_count") or 0
|
||
),
|
||
"machine_request_manifest_step_count": int(
|
||
auto_policy_details.get("machine_request_manifest_step_count") or 0
|
||
),
|
||
"lane_entry_requirement_count": int(
|
||
auto_policy_details.get("lane_entry_requirement_count") or 0
|
||
),
|
||
"reads_secret_count": int(auto_policy_details.get("reads_secret_count") or 0),
|
||
"executes_sql_count": int(auto_policy_details.get("executes_sql_count") or 0),
|
||
"writes_database_count": int(auto_policy_details.get("writes_database_count") or 0),
|
||
"primary_human_gate_count": int(
|
||
auto_policy_details.get("primary_human_gate_count") or 0
|
||
),
|
||
"manual_review_mode": auto_policy_details.get("manual_review_mode"),
|
||
"machine_verifiable": bool(auto_policy_details.get("machine_verifiable")),
|
||
"ready_for_database_apply_now": bool(
|
||
auto_policy_details.get("ready_for_database_apply_now")
|
||
),
|
||
"issues_database_apply_authorization": bool(
|
||
auto_policy_details.get("issues_database_apply_authorization")
|
||
),
|
||
"writes_database": False,
|
||
},
|
||
},
|
||
{
|
||
"key": "pchome_auto_policy_decision_preflight",
|
||
"label": "PChome auto-policy decision preflight",
|
||
"status": auto_policy_preflight.get("status") or "warning",
|
||
"summary": (
|
||
auto_policy_preflight.get("summary")
|
||
or "PChome auto-policy decision preflight has no latest check."
|
||
),
|
||
"next_machine_action": auto_policy_preflight_details.get("next_machine_action")
|
||
or (
|
||
"continue_pchome_auto_policy_decision_preflight_monitoring"
|
||
if auto_policy_preflight.get("status") == "ok"
|
||
else "refresh_pchome_auto_policy_decision_preflight"
|
||
),
|
||
"details": {
|
||
"policy": auto_policy_preflight_details.get("policy"),
|
||
"result": auto_policy_preflight_details.get("result"),
|
||
"decision_preflight_ready_count": int(
|
||
auto_policy_preflight_details.get("decision_preflight_ready_count") or 0
|
||
),
|
||
"authorization_lane_guard_ready_count": int(
|
||
auto_policy_preflight_details.get("authorization_lane_guard_ready_count") or 0
|
||
),
|
||
"decision_preflight_check_count": int(
|
||
auto_policy_preflight_details.get("decision_preflight_check_count") or 0
|
||
),
|
||
"decision_preflight_pass_count": int(
|
||
auto_policy_preflight_details.get("decision_preflight_pass_count") or 0
|
||
),
|
||
"decision_preflight_waiting_count": int(
|
||
auto_policy_preflight_details.get("decision_preflight_waiting_count") or 0
|
||
),
|
||
"decision_input_requirement_count": int(
|
||
auto_policy_preflight_details.get("decision_input_requirement_count") or 0
|
||
),
|
||
"decision_rejection_reason_count": int(
|
||
auto_policy_preflight_details.get("decision_rejection_reason_count") or 0
|
||
),
|
||
"machine_fetch_evidence_ready_count": int(
|
||
auto_policy_preflight_details.get("machine_fetch_evidence_ready_count") or 0
|
||
),
|
||
"reads_secret_count": int(
|
||
auto_policy_preflight_details.get("reads_secret_count") or 0
|
||
),
|
||
"executes_sql_count": int(
|
||
auto_policy_preflight_details.get("executes_sql_count") or 0
|
||
),
|
||
"writes_database_count": int(
|
||
auto_policy_preflight_details.get("writes_database_count") or 0
|
||
),
|
||
"primary_human_gate_count": int(
|
||
auto_policy_preflight_details.get("primary_human_gate_count") or 0
|
||
),
|
||
"manual_review_mode": auto_policy_preflight_details.get("manual_review_mode"),
|
||
"payload_source": auto_policy_preflight_details.get("payload_source"),
|
||
"execute_fetch": bool(auto_policy_preflight_details.get("execute_fetch")),
|
||
"outbound_network": bool(auto_policy_preflight_details.get("outbound_network")),
|
||
"business_data_source": bool(
|
||
auto_policy_preflight_details.get("business_data_source")
|
||
),
|
||
"ready_for_database_apply_now": bool(
|
||
auto_policy_preflight_details.get("ready_for_database_apply_now")
|
||
),
|
||
"issues_database_apply_authorization": bool(
|
||
auto_policy_preflight_details.get("issues_database_apply_authorization")
|
||
),
|
||
"writes_database": False,
|
||
},
|
||
},
|
||
{
|
||
"key": "pchome_auto_policy_decision_closeout",
|
||
"label": "PChome auto-policy decision closeout",
|
||
"status": auto_policy_closeout.get("status") or "warning",
|
||
"summary": (
|
||
auto_policy_closeout.get("summary")
|
||
or "PChome auto-policy decision closeout has no latest check."
|
||
),
|
||
"next_machine_action": auto_policy_closeout_details.get("next_machine_action")
|
||
or (
|
||
"continue_pchome_auto_policy_decision_closeout_monitoring"
|
||
if auto_policy_closeout.get("status") == "ok"
|
||
else "refresh_pchome_auto_policy_decision_closeout"
|
||
),
|
||
"details": {
|
||
"policy": auto_policy_closeout_details.get("policy"),
|
||
"result": auto_policy_closeout_details.get("result"),
|
||
"decision_closeout_ready_count": int(
|
||
auto_policy_closeout_details.get("decision_closeout_ready_count") or 0
|
||
),
|
||
"decision_preflight_ready_count": int(
|
||
auto_policy_closeout_details.get("decision_preflight_ready_count") or 0
|
||
),
|
||
"decision_closeout_check_count": int(
|
||
auto_policy_closeout_details.get("decision_closeout_check_count") or 0
|
||
),
|
||
"decision_closeout_pass_count": int(
|
||
auto_policy_closeout_details.get("decision_closeout_pass_count") or 0
|
||
),
|
||
"decision_closeout_waiting_count": int(
|
||
auto_policy_closeout_details.get("decision_closeout_waiting_count") or 0
|
||
),
|
||
"post_apply_verifier_required_count": int(
|
||
auto_policy_closeout_details.get("post_apply_verifier_required_count") or 0
|
||
),
|
||
"same_run_truth_required_count": int(
|
||
auto_policy_closeout_details.get("same_run_truth_required_count") or 0
|
||
),
|
||
"reads_secret_count": int(auto_policy_closeout_details.get("reads_secret_count") or 0),
|
||
"executes_sql_count": int(auto_policy_closeout_details.get("executes_sql_count") or 0),
|
||
"writes_database_count": int(
|
||
auto_policy_closeout_details.get("writes_database_count") or 0
|
||
),
|
||
"primary_human_gate_count": int(
|
||
auto_policy_closeout_details.get("primary_human_gate_count") or 0
|
||
),
|
||
"manual_review_mode": auto_policy_closeout_details.get("manual_review_mode"),
|
||
"payload_source": auto_policy_closeout_details.get("payload_source"),
|
||
"execute_fetch": bool(auto_policy_closeout_details.get("execute_fetch")),
|
||
"outbound_network": bool(auto_policy_closeout_details.get("outbound_network")),
|
||
"business_data_source": bool(auto_policy_closeout_details.get("business_data_source")),
|
||
"ready_for_database_apply_now": bool(
|
||
auto_policy_closeout_details.get("ready_for_database_apply_now")
|
||
),
|
||
"issues_database_apply_authorization": bool(
|
||
auto_policy_closeout_details.get("issues_database_apply_authorization")
|
||
),
|
||
"permits_future_authorization_decision_lane": bool(
|
||
auto_policy_closeout_details.get("permits_future_authorization_decision_lane")
|
||
),
|
||
"writes_database": False,
|
||
},
|
||
},
|
||
{
|
||
"key": "ai_surface_html_readback",
|
||
"label": "AI surface HTML readback",
|
||
"status": surface_readback.get("status") or "warning",
|
||
"summary": surface_readback.get("summary") or "AI surface HTML readback has no latest check.",
|
||
"next_machine_action": (
|
||
"keep_surface_readback_in_ai_smoke"
|
||
if surface_readback.get("status") == "ok"
|
||
else "repair_ai_surface_html_contract"
|
||
),
|
||
"details": {
|
||
"policy": surface_details.get("policy"),
|
||
"checked_surface_count": int(surface_details.get("checked_surface_count") or 0),
|
||
"pass_count": int(surface_details.get("pass_count") or 0),
|
||
"failed_count": int(surface_details.get("failed_count") or 0),
|
||
"forbidden_leak_count": int(surface_details.get("forbidden_leak_count") or 0),
|
||
"writes_database": False,
|
||
},
|
||
},
|
||
{
|
||
"key": "sitewide_ui_ux_agent",
|
||
"label": "Sitewide UI/UX Agent",
|
||
"status": sitewide_readback.get("status") or "warning",
|
||
"summary": sitewide_readback.get("summary") or "Sitewide UI/UX Agent has no latest check.",
|
||
"next_machine_action": (
|
||
"keep_sitewide_ui_ux_agent_monitoring"
|
||
if sitewide_readback.get("status") == "ok"
|
||
else "build_sitewide_ui_ux_repair_package"
|
||
),
|
||
"details": {
|
||
"policy": sitewide_details.get("policy"),
|
||
"template_count": int(sitewide_details.get("template_count") or 0),
|
||
"high_priority_template_count": int(sitewide_details.get("high_priority_template_count") or 0),
|
||
"compact_guardrail_count": int(sitewide_details.get("compact_guardrail_count") or 0),
|
||
"issue_surface_count": int(sitewide_details.get("issue_surface_count") or 0),
|
||
"raw_engineering_issue_count": int(sitewide_details.get("raw_engineering_issue_count") or 0),
|
||
"repair_package_status": sitewide_details.get("repair_package_status"),
|
||
"repair_action_count": int(sitewide_details.get("repair_action_count") or 0),
|
||
"writes_database": False,
|
||
},
|
||
},
|
||
{
|
||
"key": "sitewide_visual_qa",
|
||
"label": "Sitewide visual QA",
|
||
"status": visual_qa_readback.get("status") or "warning",
|
||
"summary": visual_qa_readback.get("summary") or "Sitewide visual QA has no latest check.",
|
||
"next_machine_action": (
|
||
"keep_sitewide_visual_qa_monitoring"
|
||
if visual_qa_readback.get("status") == "ok"
|
||
else "run_sitewide_visual_qa_and_publish_artifact"
|
||
),
|
||
"details": {
|
||
"policy": visual_qa_details.get("policy"),
|
||
"result_count": int(visual_qa_details.get("result_count") or 0),
|
||
"route_count": int(visual_qa_details.get("route_count") or 0),
|
||
"viewport_count": int(visual_qa_details.get("viewport_count") or 0),
|
||
"failed_count": int(visual_qa_details.get("failed_count") or 0),
|
||
"overflow_issue_count": int(visual_qa_details.get("overflow_issue_count") or 0),
|
||
"visual_offender_count": int(visual_qa_details.get("visual_offender_count") or 0),
|
||
"artifact_path": visual_qa_details.get("artifact_path"),
|
||
"artifact_generated_at": visual_qa_details.get("artifact_generated_at"),
|
||
"stale": bool(visual_qa_details.get("stale")),
|
||
"writes_database": False,
|
||
},
|
||
},
|
||
freshness_family,
|
||
{
|
||
"key": "daily_summary_delivery",
|
||
"label": "Daily summary delivery readiness",
|
||
"status": "ok" if history_count else "warning",
|
||
"summary": (
|
||
f"{len(daily_rows)} daily summary buckets available."
|
||
if history_count
|
||
else "No history is available for a daily summary yet."
|
||
),
|
||
"next_machine_action": (
|
||
"send_daily_summary_from_history"
|
||
if history_count
|
||
else "collect_smoke_before_daily_summary"
|
||
),
|
||
"details": {
|
||
"history_record_count": history_count,
|
||
"daily_bucket_count": len(daily_rows),
|
||
"send_endpoint": "/api/ai-automation/smoke/daily-summary/send",
|
||
"writes_database": False,
|
||
},
|
||
},
|
||
]
|
||
worst = max(families, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"]
|
||
primary_human_gate_count = sum(
|
||
int((item.get("details") or {}).get("primary_human_gate_count") or 0)
|
||
for item in families
|
||
)
|
||
writes_database_count = sum(
|
||
int((item.get("details") or {}).get("writes_database_count") or 0)
|
||
for item in families
|
||
)
|
||
summary = {
|
||
"ok": sum(1 for item in families if item["status"] == "ok"),
|
||
"warning": sum(1 for item in families if item["status"] == "warning"),
|
||
"critical": sum(1 for item in families if item["status"] == "critical"),
|
||
"total": len(families),
|
||
"history_record_count": history_count,
|
||
"primary_human_gate_count": primary_human_gate_count,
|
||
"writes_database_count": writes_database_count,
|
||
}
|
||
problem_families = [
|
||
item for item in families if item.get("status") in {"warning", "critical"}
|
||
]
|
||
return {
|
||
"policy": SCHEDULED_AUTOMATION_HEALTH_SUMMARY_POLICY,
|
||
"status": worst,
|
||
"version": SYSTEM_VERSION,
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"summary": summary,
|
||
"families": families,
|
||
"history": {
|
||
"counts": history.get("counts") or {},
|
||
"daily": daily_rows,
|
||
"latest": latest_history,
|
||
},
|
||
"scheduled_outputs": {
|
||
"health_summary_endpoint": "/api/ai-automation/scheduled-health-summary",
|
||
"smoke_endpoint": "/api/ai-automation/smoke",
|
||
"daily_summary_send_endpoint": "/api/ai-automation/smoke/daily-summary/send",
|
||
"telegram_send_in_preview": False,
|
||
"writes_database": False,
|
||
},
|
||
"automation_policy": {
|
||
"primary_flow": "ai_controlled",
|
||
"manual_review_mode": "exception_only",
|
||
"primary_human_gate_count": 0,
|
||
"machine_verifiable_evidence": True,
|
||
},
|
||
"next_machine_actions": [
|
||
item.get("next_machine_action")
|
||
for item in (problem_families or families[:1])
|
||
if item.get("next_machine_action")
|
||
],
|
||
"safety": {
|
||
"read_only_summary": True,
|
||
"include_current_smoke": bool(include_current_smoke or current_smoke),
|
||
"sends_telegram": False,
|
||
"writes_database": False,
|
||
"writes_database_count": 0,
|
||
"requires_production_version_truth": True,
|
||
},
|
||
}
|
||
|
||
|
||
def _daily_summary(records: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||
by_day: Dict[str, Dict[str, int]] = {}
|
||
for record in records:
|
||
day = str(record.get("generated_at") or "unknown")[:10]
|
||
bucket = by_day.setdefault(day, {"ok": 0, "warning": 0, "critical": 0, "total": 0})
|
||
status = record.get("status", "critical")
|
||
if status in bucket:
|
||
bucket[status] += 1
|
||
bucket["total"] += 1
|
||
return [
|
||
{"date": day, **counts}
|
||
for day, counts in sorted(by_day.items())[-14:]
|
||
]
|
||
|
||
|
||
def export_smoke_history_jsonl() -> Dict[str, Any]:
|
||
try:
|
||
lines = _read_history_lines()
|
||
except FileNotFoundError:
|
||
lines = []
|
||
return {
|
||
"content": "".join(lines),
|
||
"count": sum(1 for line in lines if line.strip()),
|
||
"path": _HISTORY_PATH,
|
||
}
|
||
|
||
|
||
def clear_smoke_history() -> Dict[str, Any]:
|
||
try:
|
||
cleared = _count_jsonl_lines(_HISTORY_PATH)
|
||
with _HISTORY_LOCK:
|
||
try:
|
||
os.remove(_HISTORY_PATH)
|
||
except FileNotFoundError:
|
||
cleared = 0
|
||
return {"cleared": cleared, "path": _HISTORY_PATH}
|
||
except Exception as exc:
|
||
return {"cleared": 0, "path": _HISTORY_PATH, "error": str(exc)[:300]}
|
||
|
||
|
||
def build_smoke_daily_summary_message(history_limit: int = 200) -> str:
|
||
records = _load_history(limit=history_limit)
|
||
summary = _history_summary(records)
|
||
daily_rows = summary.get("daily", [])
|
||
latest = summary.get("latest") or {}
|
||
counts = summary.get("counts") or {"ok": 0, "warning": 0, "critical": 0}
|
||
|
||
lines = [
|
||
"🤖 <b>AI 自動化 Smoke 每日摘要</b>",
|
||
"━━━━━━━━━━━━━━━━━━━━",
|
||
f"版本：<code>{escape(SYSTEM_VERSION)}</code>",
|
||
f"最近狀態：<b>{escape(str(latest.get('status', 'no_data')).upper())}</b>",
|
||
f"最近時間：{escape(str(latest.get('generated_at', '尚無紀錄')))}",
|
||
"",
|
||
"📊 <b>近期統計</b>",
|
||
f"✅ OK：{counts.get('ok', 0)}",
|
||
f"⚠️ Warning：{counts.get('warning', 0)}",
|
||
f"🚨 Critical：{counts.get('critical', 0)}",
|
||
]
|
||
|
||
if daily_rows:
|
||
lines += ["", "🗓️ <b>最近每日摘要</b>"]
|
||
for row in daily_rows[-5:]:
|
||
lines.append(
|
||
f"{escape(str(row.get('date')))}｜"
|
||
f"OK {row.get('ok', 0)} / W {row.get('warning', 0)} / C {row.get('critical', 0)}"
|
||
)
|
||
else:
|
||
lines += ["", "🗓️ 尚無 smoke history；請先開啟 /ai_automation_smoke 執行快檢。"]
|
||
|
||
problem_checks = [
|
||
item for item in latest.get("checks", [])
|
||
if item.get("status") in {"warning", "critical"}
|
||
]
|
||
if problem_checks:
|
||
lines += ["", "🚩 <b>最近異常檢查</b>"]
|
||
for item in problem_checks[:5]:
|
||
lines.append(
|
||
f"{escape(str(item.get('status', '')).upper())}｜"
|
||
f"{escape(str(item.get('name', 'unknown')))}："
|
||
f"{escape(str(item.get('summary', '')))}"
|
||
)
|
||
|
||
lines += [
|
||
"━━━━━━━━━━━━━━━━━━━━",
|
||
"入口：/ai_automation_smoke",
|
||
]
|
||
return "\n".join(lines)
|
||
|
||
|
||
def send_smoke_daily_summary(chat_ids: List[Any] | None = None) -> Dict[str, Any]:
|
||
from services.telegram_templates import send_telegram_with_result
|
||
|
||
message = build_smoke_daily_summary_message()
|
||
result = send_telegram_with_result(message, chat_ids=chat_ids)
|
||
return {
|
||
"status": "sent" if result.get("ok") else "failed",
|
||
"telegram": result,
|
||
"message_preview": message[:500],
|
||
}
|
||
|
||
|
||
def _event_router_check() -> Dict[str, Any]:
|
||
try:
|
||
from services import event_router
|
||
from services.ai_automation_metrics import snapshot
|
||
|
||
queue_count = _count_jsonl_lines(event_router._QUEUE_PATH)
|
||
metrics = snapshot()
|
||
dispatch_total = sum(
|
||
value for (metric, _labels), value in metrics.get("counters", {}).items()
|
||
if metric == "event_router_dispatch_total"
|
||
)
|
||
status = "warning" if queue_count else "ok"
|
||
summary = "EventRouter 可用，通知 queue 乾淨" if status == "ok" else "EventRouter 可用，但有待回放通知"
|
||
return _check(
|
||
"EventRouter 通知鏈",
|
||
status,
|
||
summary,
|
||
{
|
||
"dispatch_sync": callable(getattr(event_router, "dispatch_sync", None)),
|
||
"notify_failure": callable(getattr(event_router, "notify_failure", None)),
|
||
"queued_deliveries": queue_count,
|
||
"dispatch_metric_total": dispatch_total,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check("EventRouter 通知鏈", "critical", f"EventRouter smoke 失敗：{exc}")
|
||
|
||
|
||
def _row_mapping(row: Any) -> Dict[str, Any]:
|
||
if row is None:
|
||
return {}
|
||
if isinstance(row, dict):
|
||
return row
|
||
mapping = getattr(row, "_mapping", None)
|
||
if mapping is not None:
|
||
return dict(mapping)
|
||
return {}
|
||
|
||
|
||
def _create_pchome_drift_monitor_engine(database_path: str):
|
||
from sqlalchemy import create_engine
|
||
|
||
engine_kwargs: Dict[str, Any] = {}
|
||
if str(database_path).startswith(("postgresql://", "postgresql+psycopg2://", "postgres://")):
|
||
engine_kwargs["connect_args"] = {
|
||
"options": f"-c statement_timeout={_PCHOME_DRIFT_MONITOR_DB_STATEMENT_TIMEOUT_MS}"
|
||
}
|
||
return create_engine(database_path, **engine_kwargs)
|
||
|
||
|
||
def _gemini_egress_check(window_hours: int = 24) -> Dict[str, Any]:
|
||
"""Read-only runtime sentinel for unexpected Gemini spend."""
|
||
session = None
|
||
try:
|
||
from services.gemini_guard import gemini_disabled_message, is_gemini_hard_disabled
|
||
|
||
session = get_session()
|
||
since_at = datetime.now(timezone.utc) - timedelta(hours=int(window_hours))
|
||
summary_row = session.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) AS calls,
|
||
COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0) AS tokens,
|
||
COALESCE(SUM(COALESCE(cost_usd, 0)), 0) AS cost_usd,
|
||
MAX(called_at) AS last_called
|
||
FROM ai_calls
|
||
WHERE lower(provider) = 'gemini'
|
||
AND called_at >= :since_at
|
||
"""),
|
||
{"since_at": since_at},
|
||
).fetchone()
|
||
summary = _row_mapping(summary_row)
|
||
calls = int(summary.get("calls") or 0)
|
||
tokens = int(summary.get("tokens") or 0)
|
||
cost_usd = float(summary.get("cost_usd") or 0.0)
|
||
hard_disabled = is_gemini_hard_disabled()
|
||
|
||
top_rows = []
|
||
if calls:
|
||
top_rows = [
|
||
_row_mapping(row) for row in session.execute(
|
||
text("""
|
||
SELECT
|
||
caller,
|
||
model,
|
||
COUNT(*) AS calls,
|
||
COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0) AS tokens,
|
||
COALESCE(SUM(COALESCE(cost_usd, 0)), 0) AS cost_usd,
|
||
MAX(called_at) AS last_called
|
||
FROM ai_calls
|
||
WHERE lower(provider) = 'gemini'
|
||
AND called_at >= :since_at
|
||
GROUP BY caller, model
|
||
ORDER BY calls DESC, last_called DESC
|
||
LIMIT 5
|
||
"""),
|
||
{"since_at": since_at},
|
||
).fetchall()
|
||
]
|
||
|
||
details = {
|
||
"window_hours": int(window_hours),
|
||
"since_at": since_at.isoformat(timespec="seconds"),
|
||
"hard_disabled": hard_disabled,
|
||
"calls": calls,
|
||
"tokens": tokens,
|
||
"cost_usd": round(cost_usd, 6),
|
||
"last_called": str(summary.get("last_called") or ""),
|
||
"top_callers": top_rows,
|
||
"guard_reason": gemini_disabled_message("ai_smoke_gemini_egress"),
|
||
}
|
||
|
||
if calls == 0:
|
||
return _check(
|
||
"Gemini 出站費用 sentinel",
|
||
"ok",
|
||
f"最近 {window_hours}h Gemini 出站 0 次，費用 $0",
|
||
details,
|
||
)
|
||
if hard_disabled:
|
||
return _check(
|
||
"Gemini 出站費用 sentinel",
|
||
"critical",
|
||
f"Hard-disabled 狀態仍偵測到 Gemini {calls} 次 / ${cost_usd:.6f}",
|
||
details,
|
||
)
|
||
return _check(
|
||
"Gemini 出站費用 sentinel",
|
||
"warning",
|
||
f"Gemini emergency fallback 近 {window_hours}h 使用 {calls} 次 / ${cost_usd:.6f}",
|
||
details,
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"Gemini 出站費用 sentinel",
|
||
"warning",
|
||
f"Gemini 出站紀錄無法讀取：{exc}",
|
||
)
|
||
finally:
|
||
if session is not None:
|
||
session.close()
|
||
|
||
|
||
def _autoheal_check() -> Dict[str, Any]:
|
||
try:
|
||
import services.auto_heal_service as autoheal
|
||
|
||
protected = set(getattr(autoheal, "_PROTECTED_CONTAINERS", set()))
|
||
required = {"momo-db", "momo-postgres"}
|
||
missing = sorted(required - protected)
|
||
allowed_actions = sorted(getattr(autoheal, "_ALLOWED_ACTION_TYPES", set()))
|
||
status = "critical" if missing else "ok"
|
||
summary = "AutoHeal 保護資料庫容器，安全邊界存在" if status == "ok" else "AutoHeal protected resource 缺漏"
|
||
return _check(
|
||
"AutoHeal 安全邊界",
|
||
status,
|
||
summary,
|
||
{
|
||
"protected_containers": sorted(protected),
|
||
"missing_required_protection": missing,
|
||
"allowed_actions": allowed_actions,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check("AutoHeal 安全邊界", "critical", f"AutoHeal smoke 失敗：{exc}")
|
||
|
||
|
||
def _nemotron_check() -> Dict[str, Any]:
|
||
try:
|
||
import services.nemoton_dispatcher_service as nemotron
|
||
|
||
dispatcher_cls = (
|
||
getattr(nemotron, "NemotronDispatcherService", None)
|
||
or getattr(nemotron, "NemotronDispatcher", None)
|
||
)
|
||
fallback_ready = bool(dispatcher_cls and hasattr(dispatcher_cls, "_hermes_rule_fallback"))
|
||
api_key_configured = bool(getattr(nemotron, "NIM_API_KEY", ""))
|
||
call_count = getattr(nemotron, "_nim_call_count", {}).get("count", 0)
|
||
daily_limit = getattr(nemotron, "NIM_DAILY_LIMIT", 80)
|
||
if not fallback_ready:
|
||
status = "critical"
|
||
summary = "NemoTron Hermes fallback 缺失"
|
||
elif not api_key_configured:
|
||
status = "warning"
|
||
summary = "NemoTron API key 未設定，目前會走 Hermes fallback"
|
||
elif call_count >= daily_limit:
|
||
status = "warning"
|
||
summary = "NemoTron 配額已達上限，會走 Hermes fallback"
|
||
else:
|
||
status = "ok"
|
||
summary = "NemoTron 與 Hermes fallback 機制可用"
|
||
return _check(
|
||
"NemoTron fallback",
|
||
status,
|
||
summary,
|
||
{
|
||
"fallback_ready": fallback_ready,
|
||
"dispatcher_class": getattr(dispatcher_cls, "__name__", None),
|
||
"api_key_configured": api_key_configured,
|
||
"call_count": call_count,
|
||
"daily_limit": daily_limit,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check("NemoTron fallback", "critical", f"NemoTron smoke 失敗：{exc}")
|
||
|
||
|
||
def _embedding_queue_check() -> Dict[str, Any]:
|
||
session = None
|
||
try:
|
||
session = get_session()
|
||
rows = session.execute(
|
||
text("SELECT status, COUNT(*) AS count FROM embedding_retry_queue GROUP BY status")
|
||
).fetchall()
|
||
counts = {str(row._mapping["status"]): int(row._mapping["count"]) for row in rows}
|
||
pending = counts.get("pending", 0)
|
||
processing = counts.get("processing", 0)
|
||
if pending > 1000 or processing > 200:
|
||
status = "warning"
|
||
summary = "OpenClaw embedding queue backlog 偏高"
|
||
else:
|
||
status = "ok"
|
||
summary = "OpenClaw embedding queue 可讀取且 backlog 正常"
|
||
return _check(
|
||
"OpenClaw embedding queue",
|
||
status,
|
||
summary,
|
||
{"counts": counts, "pending": pending, "processing": processing},
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"OpenClaw embedding queue",
|
||
"warning",
|
||
f"Embedding queue 無法讀取，可能是 DB 離線或 migration 未套用：{exc}",
|
||
)
|
||
finally:
|
||
if session is not None:
|
||
session.close()
|
||
|
||
|
||
def _elephant_hitl_check() -> Dict[str, Any]:
|
||
try:
|
||
from services.elephant_alpha_autonomous_engine import ElephantAlphaAutonomousEngine
|
||
|
||
has_hitl = hasattr(ElephantAlphaAutonomousEngine, "_escalate_to_human")
|
||
has_timeout_guard = hasattr(ElephantAlphaAutonomousEngine, "_run_with_timeout")
|
||
api_key_configured = bool(os.getenv("OPENROUTER_API_KEY") or os.getenv("NVIDIA_API_KEY"))
|
||
if not has_hitl or not has_timeout_guard:
|
||
status = "critical"
|
||
summary = "ElephantAlpha AI 例外決策或 timeout guard 缺失"
|
||
elif not api_key_configured:
|
||
status = "warning"
|
||
summary = "ElephantAlpha AI 例外決策程式可用，但 API key 未設定"
|
||
else:
|
||
status = "ok"
|
||
summary = "ElephantAlpha AI 例外決策與 timeout guard 可用"
|
||
return _check(
|
||
"ElephantAlpha AI 例外決策",
|
||
status,
|
||
summary,
|
||
{
|
||
"hitl_method": has_hitl,
|
||
"timeout_guard": has_timeout_guard,
|
||
"api_key_configured": api_key_configured,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check("ElephantAlpha AI 例外決策", "critical", f"ElephantAlpha smoke 失敗：{exc}")
|
||
|
||
|
||
def _pchome_controlled_apply_drift_monitor_check() -> Dict[str, Any]:
|
||
"""Read-only monitor that runs the PChome controlled-apply drift verifier."""
|
||
engine = None
|
||
try:
|
||
from config import DATABASE_PATH
|
||
from services import pchome_mapping_backlog_service as backlog
|
||
|
||
engine = _create_pchome_drift_monitor_engine(DATABASE_PATH)
|
||
receipt_replay = (
|
||
backlog.build_pchome_direct_mapping_retry_candidate_exception_controlled_apply_receipt_replay_package(
|
||
materialize_artifacts=False,
|
||
engine=engine,
|
||
)
|
||
)
|
||
drift_verifier = (
|
||
backlog.build_pchome_direct_mapping_retry_candidate_exception_controlled_apply_drift_verifier_package(
|
||
engine=engine,
|
||
source_receipt_replay=receipt_replay,
|
||
materialize_artifacts=False,
|
||
)
|
||
)
|
||
summary = drift_verifier.get("summary") or {}
|
||
drift_count = int(summary.get("drift_count") or 0)
|
||
selector_count = int(summary.get("target_selector_count") or 0)
|
||
pass_count = int(summary.get("post_apply_readback_pass_count") or 0)
|
||
verified_count = int(summary.get("drift_verified_count") or 0)
|
||
artifact_count = int(summary.get("drift_verifier_artifact_materialized_count") or 0)
|
||
artifact_hash_match_count = int(summary.get("drift_verifier_artifact_hash_match_count") or 0)
|
||
writes_database_count = int(summary.get("writes_database_count") or 0)
|
||
result = str(drift_verifier.get("result") or "UNKNOWN")
|
||
receipt_result = str(receipt_replay.get("result") or "UNKNOWN")
|
||
|
||
if writes_database_count:
|
||
status = "critical"
|
||
summary_text = "PChome drift monitor 偵測到 verifier 有寫 DB 風險"
|
||
elif drift_count:
|
||
status = "critical"
|
||
summary_text = f"PChome controlled apply 偵測到 {drift_count} 筆 drift"
|
||
elif verified_count and selector_count and pass_count == selector_count:
|
||
status = "ok"
|
||
summary_text = f"PChome controlled apply drift 已驗證 {pass_count}/{selector_count}，目前 0 drift"
|
||
else:
|
||
status = "warning"
|
||
summary_text = "PChome controlled apply drift verifier 尚未達例行監控完成條件"
|
||
|
||
return _check(
|
||
"PChome 受控落地 drift monitor",
|
||
status,
|
||
summary_text,
|
||
{
|
||
"result": result,
|
||
"source_receipt_replay_result": receipt_result,
|
||
"selector_count": selector_count,
|
||
"readback_pass_count": pass_count,
|
||
"drift_count": drift_count,
|
||
"drift_verified_count": verified_count,
|
||
"artifact_count": artifact_count,
|
||
"artifact_hash_match_count": artifact_hash_match_count,
|
||
"writes_database_count": writes_database_count,
|
||
"writes_database": False,
|
||
"materialize_artifacts": False,
|
||
"requires_production_version_truth": True,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"PChome 受控落地 drift monitor",
|
||
"warning",
|
||
f"PChome drift verifier 例行監控暫時無法讀取：{exc}",
|
||
{
|
||
"writes_database": False,
|
||
"materialize_artifacts": False,
|
||
"requires_production_version_truth": True,
|
||
},
|
||
)
|
||
finally:
|
||
if engine is not None:
|
||
engine.dispose()
|
||
|
||
|
||
def _pchome_auto_policy_authorization_guard_check() -> Dict[str, Any]:
|
||
"""Read-only monitor for the PChome auto-policy DB apply guard lane."""
|
||
engine = None
|
||
try:
|
||
from config import DATABASE_PATH
|
||
from services import pchome_mapping_backlog_service as backlog
|
||
from services.ai_exception_contract import (
|
||
LEGACY_REVIEW_REQUIRED_COUNT_KEY,
|
||
PRIMARY_HUMAN_GATE_COUNT_KEY,
|
||
)
|
||
from services.pchome_revenue_growth_service import build_pchome_growth_opportunities
|
||
|
||
engine = _create_pchome_drift_monitor_engine(DATABASE_PATH)
|
||
payload = build_pchome_growth_opportunities(
|
||
engine,
|
||
limit=max(5, min(_PCHOME_AUTO_POLICY_GUARD_LIMIT, 50)),
|
||
)
|
||
payload["cache_state"] = "smoke_read_only"
|
||
guard = backlog.build_pchome_auto_policy_db_apply_authorization_lane_guard(
|
||
payload,
|
||
batch_size=max(1, min(_PCHOME_AUTO_POLICY_GUARD_BATCH_SIZE, 50)),
|
||
execute_fetch=False,
|
||
timeout_seconds=max(1, min(_PCHOME_AUTO_POLICY_GUARD_TIMEOUT_SECONDS, 30)),
|
||
)
|
||
summary = guard.get("summary") or {}
|
||
lane = guard.get("future_authorization_lane_guard") or {}
|
||
contract = guard.get("lane_transfer_contract") or {}
|
||
safety = guard.get("safety") or {}
|
||
result = str(guard.get("result") or "UNKNOWN")
|
||
|
||
guard_ready_count = int(summary.get("authorization_lane_guard_ready_count") or 0)
|
||
check_count = int(summary.get("lane_guard_check_count") or 0)
|
||
pass_count = int(summary.get("lane_guard_pass_count") or 0)
|
||
waiting_count = int(summary.get("lane_guard_waiting_count") or 0)
|
||
request_ready_count = int(summary.get("authorization_request_closeout_ready_count") or 0)
|
||
exact_field_count = int(summary.get("exact_request_payload_field_count") or 0)
|
||
manifest_step_count = int(summary.get("machine_request_manifest_step_count") or 0)
|
||
entry_requirement_count = int(summary.get("lane_entry_requirement_count") or 0)
|
||
reads_secret_count = int(summary.get("reads_secret_count") or 0)
|
||
executes_sql_count = int(summary.get("executes_sql_count") or 0)
|
||
writes_database_count = int(summary.get("writes_database_count") or 0)
|
||
primary_human_gate_count = int(summary.get(PRIMARY_HUMAN_GATE_COUNT_KEY) or 0)
|
||
manual_review_required_count = int(summary.get(LEGACY_REVIEW_REQUIRED_COUNT_KEY) or 0)
|
||
side_effect_count = reads_secret_count + executes_sql_count + writes_database_count
|
||
risk_detected = any(
|
||
[
|
||
side_effect_count,
|
||
primary_human_gate_count,
|
||
manual_review_required_count,
|
||
safety.get("reads_secret_in_preview") is not False,
|
||
safety.get("executes_sql") is not False,
|
||
safety.get("writes_database") is not False,
|
||
contract.get("writes_database") is not False,
|
||
contract.get("issues_database_apply_authorization") is not False,
|
||
lane.get("ready_for_database_apply_now") is not False,
|
||
lane.get("issues_database_apply_authorization") is not False,
|
||
]
|
||
)
|
||
guard_contract_present = (
|
||
check_count >= 12
|
||
and pass_count >= max(check_count - 1, 1)
|
||
and waiting_count <= 1
|
||
and exact_field_count == 10
|
||
and manifest_step_count == 6
|
||
and entry_requirement_count == 6
|
||
and contract.get("machine_verifiable") is True
|
||
and safety.get("manual_review_mode") == "exception_only"
|
||
)
|
||
|
||
if risk_detected:
|
||
status = "critical"
|
||
summary_text = "PChome auto-policy guard 偵測到 secret、SQL、DB write 或人工主 gate 風險"
|
||
next_machine_action = "halt_pchome_auto_policy_guard_and_inspect_risk"
|
||
elif guard_contract_present:
|
||
status = "ok"
|
||
summary_text = (
|
||
"PChome auto-policy guard 已自動讀回 no-write/no-secret package，"
|
||
f"{pass_count}/{check_count} guard checks 通過，人工主 gate=0"
|
||
)
|
||
next_machine_action = (
|
||
"collect_machine_fetch_evidence_for_authorization_guard"
|
||
if guard_ready_count == 0 or request_ready_count == 0
|
||
else "continue_pchome_auto_policy_guard_monitoring"
|
||
)
|
||
else:
|
||
status = "warning"
|
||
summary_text = "PChome auto-policy guard 尚未達自動監控契約"
|
||
next_machine_action = "refresh_pchome_auto_policy_authorization_guard"
|
||
|
||
return _check(
|
||
"PChome auto-policy authorization guard",
|
||
status,
|
||
summary_text,
|
||
{
|
||
"policy": guard.get("policy"),
|
||
"result": result,
|
||
"source_policy": guard.get("source_policy"),
|
||
"guard_ready_count": guard_ready_count,
|
||
"authorization_request_closeout_ready_count": request_ready_count,
|
||
"lane_guard_check_count": check_count,
|
||
"lane_guard_pass_count": pass_count,
|
||
"lane_guard_waiting_count": waiting_count,
|
||
"exact_request_payload_field_count": exact_field_count,
|
||
"machine_request_manifest_step_count": manifest_step_count,
|
||
"lane_entry_requirement_count": entry_requirement_count,
|
||
"reads_secret_count": reads_secret_count,
|
||
"executes_sql_count": executes_sql_count,
|
||
"writes_database_count": writes_database_count,
|
||
"primary_human_gate_count": primary_human_gate_count,
|
||
"manual_review_required_count": manual_review_required_count,
|
||
"manual_review_mode": safety.get("manual_review_mode"),
|
||
"machine_verifiable": bool(contract.get("machine_verifiable")),
|
||
"ready_for_database_apply_now": bool(lane.get("ready_for_database_apply_now")),
|
||
"issues_database_apply_authorization": bool(
|
||
lane.get("issues_database_apply_authorization")
|
||
),
|
||
"requires_fresh_production_truth": bool(
|
||
lane.get("requires_fresh_production_truth_in_same_run")
|
||
),
|
||
"execute_fetch": False,
|
||
"writes_database": False,
|
||
"next_machine_action": next_machine_action,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"PChome auto-policy authorization guard",
|
||
"warning",
|
||
f"PChome auto-policy guard 例行監控暫時無法讀取：{exc}",
|
||
{
|
||
"writes_database": False,
|
||
"writes_database_count": 0,
|
||
"primary_human_gate_count": 0,
|
||
"next_machine_action": "refresh_pchome_auto_policy_authorization_guard",
|
||
},
|
||
)
|
||
finally:
|
||
if engine is not None:
|
||
engine.dispose()
|
||
|
||
|
||
class _PChomeMachineFetchEvidenceResponse:
|
||
status_code = 200
|
||
encoding = "utf-8"
|
||
content = b"""
|
||
<html><head>
|
||
<script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "Product",
|
||
"name": "MOMO Pro machine evidence fixture",
|
||
"image": "https://cdn.example.test/pchome-auto-policy-machine-evidence.jpg",
|
||
"offers": {
|
||
"@type": "Offer",
|
||
"availability": "https://schema.org/InStock"
|
||
}
|
||
}
|
||
</script>
|
||
</head></html>
|
||
"""
|
||
|
||
|
||
def _pchome_auto_policy_machine_fetch_evidence_get(url, timeout, headers):
|
||
"""Deterministic no-network adapter for schema evidence preflight checks."""
|
||
return _PChomeMachineFetchEvidenceResponse()
|
||
|
||
|
||
def _pchome_auto_policy_decision_preflight_contract_payload() -> Dict[str, Any]:
|
||
return {
|
||
"success": True,
|
||
"system_name": "MOMO Pro",
|
||
"generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||
"cache_state": "smoke_read_only_machine_evidence_contract_fixture",
|
||
"stats": {
|
||
"candidate_count": 4,
|
||
"mapped_count": 1,
|
||
"mapping_rate": 25.0,
|
||
"needs_mapping_count": 3,
|
||
"review_candidate_count": 1,
|
||
"overall_latest_sales_date": "2026-06-24",
|
||
"overall_sales_7d": 2020234.0,
|
||
"action_counts": {
|
||
"先補商品對應": 2,
|
||
"確認候選": 1,
|
||
"放大價格優勢": 1,
|
||
},
|
||
"action_code_counts": {
|
||
"map_external_product": 2,
|
||
"review_external_candidate": 1,
|
||
"amplify_price_advantage": 1,
|
||
},
|
||
},
|
||
"opportunities": [
|
||
{
|
||
"pchome_product_id": "PCH-1",
|
||
"product_name": "Mapped product",
|
||
"sales_7d": 0,
|
||
"external_price": {"momo_sku": "M-1", "price_basis": "unit_price", "gap_pct": 12.5},
|
||
"recommended_action": {"code": "amplify_price_advantage", "label": "放大價格優勢"},
|
||
"priority_score": 75.0,
|
||
},
|
||
{
|
||
"pchome_product_id": "PCH-2",
|
||
"product_name": "Direct mapping product 40ml x2",
|
||
"sales_7d": 9800,
|
||
"pchome_price": 1200,
|
||
"external_price": None,
|
||
"recommended_action": {"code": "map_external_product", "label": "先補商品對應"},
|
||
"priority_score": 88.0,
|
||
"reason_lines": ["需要補商品對應"],
|
||
},
|
||
{
|
||
"pchome_product_id": "PCH-3",
|
||
"product_name": "Review candidate product",
|
||
"sales_7d": 1200,
|
||
"external_price": None,
|
||
"review_candidate": {
|
||
"id": 725,
|
||
"momo_sku": "5868343",
|
||
"momo_name": "MOMO candidate",
|
||
"quality_score": 94.8,
|
||
},
|
||
"recommended_action": {"code": "review_external_candidate", "label": "確認候選"},
|
||
"priority_score": 64.0,
|
||
},
|
||
{
|
||
"pchome_product_id": "PCH-4",
|
||
"product_name": "Another direct mapping product",
|
||
"sales_7d": 3100,
|
||
"external_price": None,
|
||
"recommended_action": {"code": "map_external_product", "label": "先補商品對應"},
|
||
"priority_score": 52.0,
|
||
},
|
||
],
|
||
}
|
||
|
||
|
||
def _pchome_auto_policy_decision_preflight_check() -> Dict[str, Any]:
|
||
"""Read-only monitor for machine evidence and decision preflight readiness."""
|
||
try:
|
||
from services import pchome_mapping_backlog_service as backlog
|
||
from services.ai_exception_contract import (
|
||
LEGACY_REVIEW_REQUIRED_COUNT_KEY,
|
||
PRIMARY_HUMAN_GATE_COUNT_KEY,
|
||
)
|
||
|
||
payload = _pchome_auto_policy_decision_preflight_contract_payload()
|
||
preflight = backlog.build_pchome_auto_policy_db_apply_authorization_decision_preflight(
|
||
payload,
|
||
batch_size=max(1, min(_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_BATCH_SIZE, 50)),
|
||
execute_fetch=True,
|
||
timeout_seconds=max(1, min(_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_TIMEOUT_SECONDS, 30)),
|
||
http_get=_pchome_auto_policy_machine_fetch_evidence_get,
|
||
)
|
||
summary = preflight.get("summary") or {}
|
||
decision = preflight.get("future_authorization_decision_preflight") or {}
|
||
envelope = preflight.get("decision_preflight_envelope") or {}
|
||
safety = preflight.get("safety") or {}
|
||
result = str(preflight.get("result") or "UNKNOWN")
|
||
|
||
decision_ready_count = int(
|
||
summary.get("authorization_decision_preflight_ready_count") or 0
|
||
)
|
||
lane_ready_count = int(summary.get("authorization_lane_guard_ready_count") or 0)
|
||
check_count = int(summary.get("decision_preflight_check_count") or 0)
|
||
pass_count = int(summary.get("decision_preflight_pass_count") or 0)
|
||
waiting_count = int(summary.get("decision_preflight_waiting_count") or 0)
|
||
input_requirement_count = int(summary.get("decision_input_requirement_count") or 0)
|
||
rejection_reason_count = int(summary.get("decision_rejection_reason_count") or 0)
|
||
entry_requirement_count = int(summary.get("lane_entry_requirement_count") or 0)
|
||
exact_field_count = int(summary.get("exact_request_payload_field_count") or 0)
|
||
manifest_step_count = int(summary.get("machine_request_manifest_step_count") or 0)
|
||
reads_secret_count = int(summary.get("reads_secret_count") or 0)
|
||
executes_sql_count = int(summary.get("executes_sql_count") or 0)
|
||
writes_database_count = int(summary.get("writes_database_count") or 0)
|
||
primary_human_gate_count = int(summary.get(PRIMARY_HUMAN_GATE_COUNT_KEY) or 0)
|
||
manual_review_required_count = int(summary.get(LEGACY_REVIEW_REQUIRED_COUNT_KEY) or 0)
|
||
side_effect_count = reads_secret_count + executes_sql_count + writes_database_count
|
||
risk_detected = any(
|
||
[
|
||
side_effect_count,
|
||
primary_human_gate_count,
|
||
manual_review_required_count,
|
||
safety.get("reads_secret_in_preview") is not False,
|
||
safety.get("executes_sql") is not False,
|
||
safety.get("writes_database") is not False,
|
||
envelope.get("issues_database_apply_authorization") is not False,
|
||
envelope.get("ready_for_database_apply_now") is not False,
|
||
decision.get("ready_for_database_apply_now") is not False,
|
||
decision.get("issues_database_apply_authorization") is not False,
|
||
]
|
||
)
|
||
preflight_contract_present = (
|
||
decision_ready_count == 1
|
||
and lane_ready_count == 1
|
||
and check_count >= 12
|
||
and pass_count == check_count
|
||
and waiting_count == 0
|
||
and input_requirement_count == 8
|
||
and rejection_reason_count == 10
|
||
and entry_requirement_count == 6
|
||
and exact_field_count == 10
|
||
and manifest_step_count == 6
|
||
and decision.get("ready_for_future_authorization_decision") is True
|
||
and envelope.get("allows_authorization_decision_in_future_lane") is True
|
||
and safety.get("manual_review_mode") == "exception_only"
|
||
)
|
||
|
||
if risk_detected:
|
||
status = "critical"
|
||
summary_text = "PChome auto-policy decision preflight 偵測到 secret、SQL、DB write 或人工主 gate 風險"
|
||
next_machine_action = "halt_pchome_auto_policy_decision_preflight_and_inspect_risk"
|
||
elif preflight_contract_present:
|
||
status = "ok"
|
||
summary_text = (
|
||
"PChome auto-policy decision preflight 已用本機 machine evidence 自動通過 "
|
||
f"{pass_count}/{check_count} checks，外部網路=0、人工主 gate=0"
|
||
)
|
||
next_machine_action = "continue_to_pchome_auto_policy_decision_closeout_lane"
|
||
else:
|
||
status = "warning"
|
||
summary_text = "PChome auto-policy decision preflight 尚未達自動監控契約"
|
||
next_machine_action = "refresh_pchome_auto_policy_machine_fetch_evidence_preflight"
|
||
|
||
return _check(
|
||
"PChome auto-policy decision preflight",
|
||
status,
|
||
summary_text,
|
||
{
|
||
"policy": preflight.get("policy"),
|
||
"result": result,
|
||
"source_policy": preflight.get("source_policy"),
|
||
"decision_preflight_ready_count": decision_ready_count,
|
||
"authorization_lane_guard_ready_count": lane_ready_count,
|
||
"decision_preflight_check_count": check_count,
|
||
"decision_preflight_pass_count": pass_count,
|
||
"decision_preflight_waiting_count": waiting_count,
|
||
"decision_input_requirement_count": input_requirement_count,
|
||
"decision_rejection_reason_count": rejection_reason_count,
|
||
"lane_entry_requirement_count": entry_requirement_count,
|
||
"exact_request_payload_field_count": exact_field_count,
|
||
"machine_request_manifest_step_count": manifest_step_count,
|
||
"reads_secret_count": reads_secret_count,
|
||
"executes_sql_count": executes_sql_count,
|
||
"writes_database_count": writes_database_count,
|
||
"primary_human_gate_count": primary_human_gate_count,
|
||
"manual_review_required_count": manual_review_required_count,
|
||
"manual_review_mode": safety.get("manual_review_mode"),
|
||
"machine_fetch_evidence_ready_count": 1 if preflight_contract_present else 0,
|
||
"machine_fetch_evidence_source": "local_schema_fixture",
|
||
"payload_source": "local_contract_fixture",
|
||
"http_get_adapter": "_pchome_auto_policy_machine_fetch_evidence_get",
|
||
"execute_fetch": True,
|
||
"outbound_network": False,
|
||
"business_data_source": False,
|
||
"ready_for_database_apply_now": bool(decision.get("ready_for_database_apply_now")),
|
||
"issues_database_apply_authorization": bool(
|
||
decision.get("issues_database_apply_authorization")
|
||
),
|
||
"requires_fresh_production_truth": bool(
|
||
decision.get("requires_fresh_production_truth_in_same_run")
|
||
),
|
||
"writes_database": False,
|
||
"next_machine_action": next_machine_action,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"PChome auto-policy decision preflight",
|
||
"warning",
|
||
f"PChome auto-policy decision preflight 例行監控暫時無法讀取：{exc}",
|
||
{
|
||
"writes_database": False,
|
||
"writes_database_count": 0,
|
||
"primary_human_gate_count": 0,
|
||
"execute_fetch": True,
|
||
"outbound_network": False,
|
||
"business_data_source": False,
|
||
"payload_source": "local_contract_fixture",
|
||
"next_machine_action": "refresh_pchome_auto_policy_machine_fetch_evidence_preflight",
|
||
},
|
||
)
|
||
|
||
|
||
def _pchome_auto_policy_decision_closeout_check() -> Dict[str, Any]:
|
||
"""Read-only monitor for the future authorization decision closeout package."""
|
||
try:
|
||
from services import pchome_mapping_backlog_service as backlog
|
||
from services.ai_exception_contract import (
|
||
LEGACY_REVIEW_REQUIRED_COUNT_KEY,
|
||
PRIMARY_HUMAN_GATE_COUNT_KEY,
|
||
)
|
||
|
||
payload = _pchome_auto_policy_decision_preflight_contract_payload()
|
||
closeout = backlog.build_pchome_auto_policy_db_apply_authorization_decision_closeout(
|
||
payload,
|
||
batch_size=max(1, min(_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_BATCH_SIZE, 50)),
|
||
execute_fetch=True,
|
||
timeout_seconds=max(1, min(_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_TIMEOUT_SECONDS, 30)),
|
||
http_get=_pchome_auto_policy_machine_fetch_evidence_get,
|
||
)
|
||
summary = closeout.get("summary") or {}
|
||
decision = closeout.get("future_authorization_decision_closeout") or {}
|
||
package = closeout.get("future_authorization_decision_package") or {}
|
||
contract = closeout.get("decision_closeout_contract") or {}
|
||
safety = closeout.get("safety") or {}
|
||
result = str(closeout.get("result") or "UNKNOWN")
|
||
|
||
closeout_ready_count = int(summary.get("authorization_decision_closeout_ready_count") or 0)
|
||
preflight_ready_count = int(
|
||
summary.get("authorization_decision_preflight_ready_count") or 0
|
||
)
|
||
check_count = int(summary.get("decision_closeout_check_count") or 0)
|
||
pass_count = int(summary.get("decision_closeout_pass_count") or 0)
|
||
waiting_count = int(summary.get("decision_closeout_waiting_count") or 0)
|
||
input_requirement_count = int(summary.get("decision_input_requirement_count") or 0)
|
||
rejection_reason_count = int(summary.get("decision_rejection_reason_count") or 0)
|
||
verifier_required_count = int(summary.get("post_apply_verifier_required_count") or 0)
|
||
same_run_truth_count = int(summary.get("same_run_truth_required_count") or 0)
|
||
reads_secret_count = int(summary.get("reads_secret_count") or 0)
|
||
executes_sql_count = int(summary.get("executes_sql_count") or 0)
|
||
writes_database_count = int(summary.get("writes_database_count") or 0)
|
||
primary_human_gate_count = int(summary.get(PRIMARY_HUMAN_GATE_COUNT_KEY) or 0)
|
||
manual_review_required_count = int(summary.get(LEGACY_REVIEW_REQUIRED_COUNT_KEY) or 0)
|
||
side_effect_count = reads_secret_count + executes_sql_count + writes_database_count
|
||
risk_detected = any(
|
||
[
|
||
side_effect_count,
|
||
primary_human_gate_count,
|
||
manual_review_required_count,
|
||
safety.get("reads_secret_in_preview") is not False,
|
||
safety.get("executes_sql") is not False,
|
||
safety.get("writes_database") is not False,
|
||
contract.get("issues_database_apply_authorization") is not False,
|
||
contract.get("ready_for_database_apply_now") is not False,
|
||
contract.get("writes_database") is not False,
|
||
package.get("ready_for_database_apply_now") is not False,
|
||
package.get("issues_database_apply_authorization") is not False,
|
||
decision.get("ready_for_database_apply_now") is not False,
|
||
decision.get("issues_database_apply_authorization") is not False,
|
||
]
|
||
)
|
||
closeout_contract_present = (
|
||
closeout_ready_count == 1
|
||
and preflight_ready_count == 1
|
||
and check_count >= 12
|
||
and pass_count == check_count
|
||
and waiting_count == 0
|
||
and input_requirement_count == 8
|
||
and rejection_reason_count == 10
|
||
and verifier_required_count == 1
|
||
and same_run_truth_count == 1
|
||
and decision.get("ready_for_future_authorization_decision_closeout") is True
|
||
and package.get("ready_for_future_authorization_decision_package") is True
|
||
and package.get("requires_post_apply_verifier") is True
|
||
and package.get("requires_fresh_production_truth_in_same_run") is True
|
||
and contract.get("machine_verifiable") is True
|
||
and contract.get("permits_future_authorization_decision_lane") is True
|
||
and safety.get("manual_review_mode") == "exception_only"
|
||
)
|
||
|
||
if risk_detected:
|
||
status = "critical"
|
||
summary_text = "PChome auto-policy decision closeout 偵測到 secret、SQL、DB write 或人工主 gate 風險"
|
||
next_machine_action = "halt_pchome_auto_policy_decision_closeout_and_inspect_risk"
|
||
elif closeout_contract_present:
|
||
status = "ok"
|
||
summary_text = (
|
||
"PChome auto-policy decision closeout 已自動完成 "
|
||
f"{pass_count}/{check_count} checks，post-apply verifier 已綁定，DB apply authorization=0"
|
||
)
|
||
next_machine_action = "continue_to_pchome_auto_policy_authorization_issuer_gate"
|
||
else:
|
||
status = "warning"
|
||
summary_text = "PChome auto-policy decision closeout 尚未達自動監控契約"
|
||
next_machine_action = "refresh_pchome_auto_policy_decision_closeout"
|
||
|
||
return _check(
|
||
"PChome auto-policy decision closeout",
|
||
status,
|
||
summary_text,
|
||
{
|
||
"policy": closeout.get("policy"),
|
||
"result": result,
|
||
"source_policy": closeout.get("source_policy"),
|
||
"decision_closeout_ready_count": closeout_ready_count,
|
||
"decision_preflight_ready_count": preflight_ready_count,
|
||
"decision_closeout_check_count": check_count,
|
||
"decision_closeout_pass_count": pass_count,
|
||
"decision_closeout_waiting_count": waiting_count,
|
||
"decision_input_requirement_count": input_requirement_count,
|
||
"decision_rejection_reason_count": rejection_reason_count,
|
||
"post_apply_verifier_required_count": verifier_required_count,
|
||
"same_run_truth_required_count": same_run_truth_count,
|
||
"reads_secret_count": reads_secret_count,
|
||
"executes_sql_count": executes_sql_count,
|
||
"writes_database_count": writes_database_count,
|
||
"primary_human_gate_count": primary_human_gate_count,
|
||
"manual_review_required_count": manual_review_required_count,
|
||
"manual_review_mode": safety.get("manual_review_mode"),
|
||
"payload_source": "local_contract_fixture",
|
||
"machine_fetch_evidence_source": "local_schema_fixture",
|
||
"http_get_adapter": "_pchome_auto_policy_machine_fetch_evidence_get",
|
||
"execute_fetch": True,
|
||
"outbound_network": False,
|
||
"business_data_source": False,
|
||
"ready_for_database_apply_now": bool(package.get("ready_for_database_apply_now")),
|
||
"issues_database_apply_authorization": bool(
|
||
package.get("issues_database_apply_authorization")
|
||
),
|
||
"permits_future_authorization_decision_lane": bool(
|
||
contract.get("permits_future_authorization_decision_lane")
|
||
),
|
||
"requires_post_apply_verifier": bool(package.get("requires_post_apply_verifier")),
|
||
"requires_fresh_production_truth": bool(
|
||
package.get("requires_fresh_production_truth_in_same_run")
|
||
),
|
||
"writes_database": False,
|
||
"next_machine_action": next_machine_action,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"PChome auto-policy decision closeout",
|
||
"warning",
|
||
f"PChome auto-policy decision closeout 例行監控暫時無法讀取：{exc}",
|
||
{
|
||
"writes_database": False,
|
||
"writes_database_count": 0,
|
||
"primary_human_gate_count": 0,
|
||
"execute_fetch": True,
|
||
"outbound_network": False,
|
||
"business_data_source": False,
|
||
"payload_source": "local_contract_fixture",
|
||
"next_machine_action": "refresh_pchome_auto_policy_decision_closeout",
|
||
},
|
||
)
|
||
|
||
|
||
def _ai_surface_html_readback_check() -> Dict[str, Any]:
|
||
"""Read-only AI product surface guardrail readback."""
|
||
try:
|
||
from services.ai_surface_html_readback_service import build_ai_surface_html_readback
|
||
|
||
readback = build_ai_surface_html_readback()
|
||
summary = readback.get("summary") or {}
|
||
checked_count = int(summary.get("checked_surface_count") or 0)
|
||
pass_count = int(summary.get("pass_count") or 0)
|
||
failed_count = int(summary.get("failed_count") or 0)
|
||
leak_count = int(summary.get("forbidden_leak_count") or 0)
|
||
status = "ok" if readback.get("status") == "ok" else "critical"
|
||
summary_text = (
|
||
f"AI surface HTML readback 通過 {pass_count}/{checked_count}，無產品面 raw wording"
|
||
if status == "ok"
|
||
else f"AI surface HTML readback 偵測 {failed_count} 個頁面退化 / {leak_count} 個 raw wording"
|
||
)
|
||
return _check(
|
||
"AI surface HTML readback",
|
||
status,
|
||
summary_text,
|
||
{
|
||
"policy": readback.get("policy"),
|
||
"checked_surface_count": checked_count,
|
||
"pass_count": pass_count,
|
||
"failed_count": failed_count,
|
||
"forbidden_leak_count": leak_count,
|
||
"next_machine_action": readback.get("next_machine_action"),
|
||
"writes_database": False,
|
||
"primary_human_gate_count": 0,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"AI surface HTML readback",
|
||
"critical",
|
||
f"AI surface HTML readback 無法執行：{exc}",
|
||
{
|
||
"writes_database": False,
|
||
"primary_human_gate_count": 0,
|
||
},
|
||
)
|
||
|
||
|
||
def _sitewide_ui_ux_agent_check() -> Dict[str, Any]:
|
||
"""Read-only sitewide professional UI/UX regression sentinel."""
|
||
try:
|
||
from services.ai_surface_html_readback_service import (
|
||
build_sitewide_ui_ux_agent_inventory,
|
||
build_sitewide_ui_ux_repair_package,
|
||
)
|
||
|
||
inventory = build_sitewide_ui_ux_agent_inventory()
|
||
repair_package = build_sitewide_ui_ux_repair_package(source_inventory=inventory, limit=5)
|
||
summary = inventory.get("summary") or {}
|
||
package_summary = repair_package.get("summary") or {}
|
||
issue_count = int(summary.get("issue_surface_count") or 0)
|
||
raw_issue_count = int(summary.get("raw_engineering_issue_count") or 0)
|
||
write_count = int(summary.get("writes_database_count") or 0) + int(
|
||
package_summary.get("writes_database_count") or 0
|
||
)
|
||
repair_action_count = int(package_summary.get("controlled_action_count") or 0)
|
||
status = "ok"
|
||
if write_count:
|
||
status = "critical"
|
||
elif issue_count or raw_issue_count or repair_action_count:
|
||
status = "critical"
|
||
summary_text = (
|
||
"Sitewide UI/UX Agent ok/no-op，整站專業化 surface 無退化"
|
||
if status == "ok"
|
||
else (
|
||
f"Sitewide UI/UX Agent 偵測 {issue_count} 個待修 surface、"
|
||
f"{raw_issue_count} 個 raw wording、{repair_action_count} 個待套用 action"
|
||
)
|
||
)
|
||
return _check(
|
||
"Sitewide UI/UX Agent readback",
|
||
status,
|
||
summary_text,
|
||
{
|
||
"policy": inventory.get("policy"),
|
||
"inventory_status": inventory.get("status"),
|
||
"template_count": int(summary.get("template_count") or 0),
|
||
"high_priority_template_count": int(summary.get("high_priority_template_count") or 0),
|
||
"compact_guardrail_count": int(summary.get("compact_guardrail_count") or 0),
|
||
"issue_surface_count": issue_count,
|
||
"raw_engineering_issue_count": raw_issue_count,
|
||
"next_machine_action": inventory.get("next_machine_action"),
|
||
"repair_package_policy": repair_package.get("policy"),
|
||
"repair_package_status": repair_package.get("status"),
|
||
"repair_action_count": repair_action_count,
|
||
"repair_next_machine_action": repair_package.get("next_machine_action"),
|
||
"writes_database": False,
|
||
"writes_database_count": write_count,
|
||
"primary_human_gate_count": 0,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"Sitewide UI/UX Agent readback",
|
||
"critical",
|
||
f"Sitewide UI/UX Agent 無法執行：{exc}",
|
||
{
|
||
"writes_database": False,
|
||
"primary_human_gate_count": 0,
|
||
},
|
||
)
|
||
|
||
|
||
def _sitewide_visual_qa_check() -> Dict[str, Any]:
|
||
"""Read-only sentinel for latest desktop/tablet/mobile visual QA artifact."""
|
||
try:
|
||
from services.ai_surface_html_readback_service import build_sitewide_visual_qa_readback
|
||
|
||
readback = build_sitewide_visual_qa_readback()
|
||
summary = readback.get("summary") or {}
|
||
status = readback.get("status") or "warning"
|
||
failed_count = int(summary.get("failed_count") or 0)
|
||
route_count = int(summary.get("route_count") or 0)
|
||
viewport_count = int(summary.get("viewport_count") or 0)
|
||
summary_text = (
|
||
f"Sitewide visual QA ok，{route_count} routes x {viewport_count} viewports 無 overflow 退化"
|
||
if status == "ok"
|
||
else (
|
||
f"Sitewide visual QA 需更新或修復：failed={failed_count}、"
|
||
f"routes={route_count}、stale={bool(summary.get('stale'))}"
|
||
)
|
||
)
|
||
return _check(
|
||
"Sitewide visual QA readback",
|
||
status,
|
||
summary_text,
|
||
{
|
||
"policy": readback.get("policy"),
|
||
"result_count": int(summary.get("result_count") or 0),
|
||
"route_count": route_count,
|
||
"viewport_count": viewport_count,
|
||
"pass_count": int(summary.get("pass_count") or 0),
|
||
"failed_count": failed_count,
|
||
"overflow_issue_count": int(summary.get("overflow_issue_count") or 0),
|
||
"visual_offender_count": int(summary.get("visual_offender_count") or 0),
|
||
"artifact_path": readback.get("artifact_path"),
|
||
"artifact_exists": bool(readback.get("artifact_exists")),
|
||
"artifact_generated_at": readback.get("artifact_generated_at"),
|
||
"age_hours": summary.get("age_hours"),
|
||
"stale": bool(summary.get("stale")),
|
||
"next_machine_action": readback.get("next_machine_action"),
|
||
"writes_database": False,
|
||
"writes_database_count": 0,
|
||
"primary_human_gate_count": 0,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"Sitewide visual QA readback",
|
||
"critical",
|
||
f"Sitewide visual QA readback 無法執行：{exc}",
|
||
{
|
||
"writes_database": False,
|
||
"primary_human_gate_count": 0,
|
||
},
|
||
)
|
||
|
||
|
||
def _latest_sitewide_visual_qa_readback_check() -> Dict[str, Any]:
|
||
"""Read the latest visual QA artifact for scheduled summaries without running smoke."""
|
||
try:
|
||
from services.ai_surface_html_readback_service import build_sitewide_visual_qa_readback
|
||
|
||
readback = build_sitewide_visual_qa_readback()
|
||
summary = readback.get("summary") or {}
|
||
status = readback.get("status") or "warning"
|
||
route_count = int(summary.get("route_count") or 0)
|
||
viewport_count = int(summary.get("viewport_count") or 0)
|
||
failed_count = int(summary.get("failed_count") or 0)
|
||
return _check(
|
||
"Sitewide visual QA readback",
|
||
status,
|
||
(
|
||
f"Sitewide visual QA ok，{route_count} routes x {viewport_count} viewports 無 overflow 退化"
|
||
if status == "ok"
|
||
else (
|
||
f"Sitewide visual QA 需更新或修復：failed={failed_count}、"
|
||
f"routes={route_count}、stale={bool(summary.get('stale'))}"
|
||
)
|
||
),
|
||
{
|
||
"policy": readback.get("policy"),
|
||
"result_count": int(summary.get("result_count") or 0),
|
||
"route_count": route_count,
|
||
"viewport_count": viewport_count,
|
||
"pass_count": int(summary.get("pass_count") or 0),
|
||
"failed_count": failed_count,
|
||
"overflow_issue_count": int(summary.get("overflow_issue_count") or 0),
|
||
"visual_offender_count": int(summary.get("visual_offender_count") or 0),
|
||
"artifact_path": readback.get("artifact_path"),
|
||
"artifact_exists": bool(readback.get("artifact_exists")),
|
||
"artifact_generated_at": readback.get("artifact_generated_at"),
|
||
"age_hours": summary.get("age_hours"),
|
||
"stale": bool(summary.get("stale")),
|
||
"next_machine_action": readback.get("next_machine_action"),
|
||
"writes_database": False,
|
||
"writes_database_count": 0,
|
||
"primary_human_gate_count": 0,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
return _check(
|
||
"Sitewide visual QA readback",
|
||
"warning",
|
||
f"Sitewide visual QA artifact fallback 無法讀取：{exc}",
|
||
{
|
||
"writes_database": False,
|
||
"primary_human_gate_count": 0,
|
||
},
|
||
)
|
||
|
||
|
||
def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]:
|
||
checks: List[Dict[str, Any]] = [
|
||
_event_router_check(),
|
||
_gemini_egress_check(),
|
||
_autoheal_check(),
|
||
_nemotron_check(),
|
||
_embedding_queue_check(),
|
||
_elephant_hitl_check(),
|
||
_pchome_controlled_apply_drift_monitor_check(),
|
||
_pchome_auto_policy_authorization_guard_check(),
|
||
_pchome_auto_policy_decision_preflight_check(),
|
||
_pchome_auto_policy_decision_closeout_check(),
|
||
_ai_surface_html_readback_check(),
|
||
_sitewide_ui_ux_agent_check(),
|
||
_sitewide_visual_qa_check(),
|
||
]
|
||
worst = max(checks, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"]
|
||
result = {
|
||
"status": worst,
|
||
"version": SYSTEM_VERSION,
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"checks": checks,
|
||
"summary": {
|
||
"ok": sum(1 for item in checks if item["status"] == "ok"),
|
||
"warning": sum(1 for item in checks if item["status"] == "warning"),
|
||
"critical": sum(1 for item in checks if item["status"] == "critical"),
|
||
"total": len(checks),
|
||
},
|
||
}
|
||
if record_history:
|
||
try:
|
||
_append_history(result)
|
||
except Exception as exc:
|
||
result["history_error"] = str(exc)[:300]
|
||
result["history"] = _history_summary(_load_history(limit=history_limit))
|
||
return result
|