feat(ai): automate agent integration truth and RAG canary

tests/test_ai_agent_product_integration_schema_contract.py

@@ -0,0 +1,17 @@
+def test_runtime_telemetry_uses_rag_query_log_queried_at(monkeypatch):
+    from services import ai_agent_product_integration_service as service
+
+    sql_seen = []
+
+    def fake_query(sql, _params):
+        sql_seen.append(sql)
+        return [], None
+
+    monkeypatch.setattr(service, "_query_rows", fake_query)
+    service._collect_runtime_telemetry(
+        service.datetime.now(service.timezone.utc)
+    )
+
+    rag_sql = next(sql for sql in sql_seen if "FROM rag_query_log" in sql)
+    assert "queried_at >= :since_at" in rag_sql
+    assert "created_at >= :since_at" not in rag_sql

tests/test_ai_agent_product_integration_service.py

@@ -0,0 +1,159 @@
+import json
+import subprocess
+import sys
+
+
+def _telemetry(*, all_active):
+    from services.ai_agent_product_integration_service import AGENT_SPECS
+
+    agents = {}
+    for index, key in enumerate(AGENT_SPECS):
+        active = all_active or index < 2
+        agents[key] = {
+            "calls": 5 if active else 0,
+            "errors": 0,
+            "rag_hits": 1 if active else 0,
+            "fallbacks": 0,
+            "mcp_calls": 1 if all_active else 0,
+            "rag_queries": 1 if all_active else 0,
+            "rag_query_hits": 1 if all_active else 0,
+            "ai_insights": 1 if active else 0,
+            "action_plans": 1 if active else 0,
+            "executed_action_plans": 1 if all_active else 0,
+            "last_called": "2026-07-17T00:00:00+00:00" if active else None,
+        }
+    return {
+        "agents": agents,
+        "unmatched_ai_callers": [],
+        "totals": {
+            "ai_calls": sum(item["calls"] for item in agents.values()),
+            "ai_errors": 0,
+            "mcp_calls": 4 if all_active else 0,
+            "rag_queries": 4 if all_active else 0,
+            "rag_query_hits": 4 if all_active else 0,
+            "ai_insights": 4,
+            "action_plans": 4,
+            "executed_action_plans": 4 if all_active else 0,
+            "action_outcomes": 2 if all_active else 0,
+            "heal_logs": 2 if all_active else 0,
+            "heal_success": 2 if all_active else 0,
+            "verified_retry_or_rollback_incidents": 1 if all_active else 0,
+        },
+        "embedding_retry_queue": {"done": 8},
+        "read_errors": [],
+    }
+
+
+def _external_runtime(*, enabled):
+    return {
+        "runtime": {
+            "mcp": {"enabled": enabled},
+            "rag": {"enabled": enabled},
+            "pixelrag": {"enabled": True, "platform_count": 8},
+        }
+    }
+
+
+def _canary(*, passed):
+    return {"latest_execution": {"canary_passed": passed}}
+
+
+def test_ai_agent_integration_reports_production_like_partial_truth(monkeypatch):
+    from services import ai_agent_product_integration_service as service
+
+    monkeypatch.setattr(
+        service,
+        "_collect_runtime_telemetry",
+        lambda _since: _telemetry(all_active=False),
+    )
+    monkeypatch.setattr(
+        service,
+        "build_external_mcp_rag_integration_readback",
+        lambda: _external_runtime(enabled=False),
+    )
+    monkeypatch.setattr(
+        service,
+        "run_internal_rag_candidate_canary",
+        lambda **_kwargs: _canary(passed=False),
+    )
+    payload = service.build_ai_agent_product_integration_readback(window_hours=168)
+
+    assert payload["status"] == "partially_integrated"
+    assert payload["completion"]["source_wired_agents"] == 4
+    assert payload["completion"]["runtime_active_agents"] == 2
+    assert payload["completion"]["full_product_integration"] is False
+    assert "mcp_router_runtime_disabled" in payload["blockers"]
+    assert "rag_runtime_disabled" in payload["blockers"]
+    assert "尚未完整整合" in payload["answer_to_owner"]
+
+
+def test_ai_agent_integration_requires_full_runtime_closure(monkeypatch):
+    from services import ai_agent_product_integration_service as service
+
+    monkeypatch.setattr(
+        service,
+        "_collect_runtime_telemetry",
+        lambda _since: _telemetry(all_active=True),
+    )
+    monkeypatch.setattr(
+        service,
+        "build_external_mcp_rag_integration_readback",
+        lambda: _external_runtime(enabled=True),
+    )
+    monkeypatch.setattr(
+        service,
+        "run_internal_rag_candidate_canary",
+        lambda **_kwargs: _canary(passed=True),
+    )
+    payload = service.build_ai_agent_product_integration_readback()
+
+    assert payload["status"] == "fully_integrated"
+    assert payload["completion"]["runtime_active_agents"] == 4
+    assert payload["completion"]["closure_stage_passed"] == 9
+    assert payload["completion"]["full_product_integration"] is True
+    assert payload["blockers"] == []
+
+
+def test_ai_agent_integration_route_returns_truth(monkeypatch):
+    from flask import Flask
+    from routes import system_public_routes as routes
+    from services import ai_agent_product_integration_service as service
+
+    monkeypatch.setattr(
+        service,
+        "build_ai_agent_product_integration_readback",
+        lambda **kwargs: {
+            "success": True,
+            "policy": service.POLICY,
+            "status": "partially_integrated",
+            "window_hours": kwargs["window_hours"],
+        },
+    )
+    app = Flask(__name__)
+    with app.test_request_context(
+        "/api/ai-automation/agent-product-integration?window_hours=72"
+    ):
+        response = routes.ai_automation_agent_product_integration_api.__wrapped__()
+    payload = response.get_json()
+
+    assert payload["policy"] == "runtime_truth_ai_agent_product_integration_v1"
+    assert payload["window_hours"] == 72
+
+
+def test_ai_agent_integration_cli_outputs_json():
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "scripts/ops/report_ai_agent_product_integration.py",
+            "--window-hours",
+            "1",
+        ],
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+    assert completed.returncode == 0
+    payload = json.loads(completed.stdout)
+    assert payload["policy"] == "runtime_truth_ai_agent_product_integration_v1"
+    assert payload["controlled_apply"]["database_write"] is False

tests/test_ai_automation_smoke_service.py

@@ -1309,7 +1309,9 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch):
     monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok"))
     monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok"))
     monkeypatch.setattr(smoke, "_security_governance_review_check", lambda: smoke._check("security governance", "ok", "ok"))
+    monkeypatch.setattr(smoke, "_ai_agent_product_integration_check", lambda: smoke._check("agent integration", "ok", "ok"))
     monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok"))
+    monkeypatch.setattr(smoke, "_internal_rag_candidate_canary_check", lambda: smoke._check("internal rag canary", "ok", "ok"))
     monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok"))
     monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok"))
     monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok"))
@@ -1327,7 +1329,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch):
     result = smoke.collect_ai_automation_smoke(record_history=False)
 
     assert result["status"] == "critical"
-    assert result["summary"] == {"ok": 43, "warning": 1, "critical": 1, "total": 45}
+    assert result["summary"] == {"ok": 45, "warning": 1, "critical": 1, "total": 47}
 
 
 def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch):
@@ -3855,7 +3857,9 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat
     monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok"))
     monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok"))
     monkeypatch.setattr(smoke, "_security_governance_review_check", lambda: smoke._check("security governance", "ok", "ok"))
+    monkeypatch.setattr(smoke, "_ai_agent_product_integration_check", lambda: smoke._check("agent integration", "ok", "ok"))
     monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok"))
+    monkeypatch.setattr(smoke, "_internal_rag_candidate_canary_check", lambda: smoke._check("internal rag canary", "ok", "ok"))
     monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok"))
     monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok"))
     monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok"))
@@ -6647,11 +6651,21 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch):
         "ok",
         "security governance ok",
     ))
+    monkeypatch.setattr(smoke, "_ai_agent_product_integration_check", lambda: smoke._check(
+        "AI Agent product integration truth",
+        "ok",
+        "agent integration ok",
+    ))
     monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check(
         "External MCP/RAG integration readback",
         "ok",
         "external mcp rag ok",
     ))
+    monkeypatch.setattr(smoke, "_internal_rag_candidate_canary_check", lambda: smoke._check(
+        "Internal RAG candidate canary",
+        "ok",
+        "internal rag canary ok",
+    ))
     monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check(
         "PixelRAG RAG candidate replay",
         "ok",
@@ -6732,7 +6746,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch):
         item for item in result["checks"]
         if item["name"] == "Sitewide visual QA readback"
     )
-    assert result["summary"]["total"] == 45
+    assert result["summary"]["total"] == 47
     assert surface_check["status"] == "ok"
     assert surface_check["details"]["checked_surface_count"] == 10
     assert sitewide_check["status"] == "ok"

tests/test_internal_rag_candidate_canary_service.py

@@ -0,0 +1,227 @@
+import json
+import inspect
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+
+def _write_candidate_receipt(root, *, signature=None, platform="shopee_tw"):
+    from services.rag_service import get_embedding_signature
+
+    embedding_signature = signature or get_embedding_signature()
+    manifest_id = "shopee-canary-001"
+    target = root / platform / manifest_id
+    target.mkdir(parents=True)
+    payload = {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "worker_status": "executed_marketplace_candidate_knowledge_replay_ready",
+        "platform": platform,
+        "manifest_id": manifest_id,
+        "writes_database": False,
+        "writes_database_count": 0,
+        "writes_ai_insights": False,
+        "writes_price_tables": False,
+        "candidate_knowledge_replay": {
+            "candidate_knowledge_replay_version": "pixelrag_marketplace_candidate_knowledge_replay_v1",
+            "embedding_signature_contract": {"embedding_signature": embedding_signature},
+            "candidate_knowledge_contracts": [{
+                "candidate_id": f"{platform}:{manifest_id}:0",
+                "candidate_knowledge_fingerprint": "candidate-fingerprint-001",
+                "candidate_knowledge_text": (
+                    f"platform={platform} | manifest_id={manifest_id} | "
+                    "candidate_id=0 | product evidence"
+                ),
+                "ready_for_internal_rag_candidate_replay": True,
+                "ready_for_ai_insights_write": False,
+                "ready_for_price_table_write": False,
+            }],
+        },
+    }
+    path = target / "marketplace_candidate_knowledge_replay_receipt.json"
+    path.write_text(json.dumps(payload), encoding="utf-8")
+    return path
+
+
+def test_internal_rag_canary_dry_run_never_calls_model_or_database(tmp_path, monkeypatch):
+    from services import internal_rag_candidate_canary_service as service
+
+    source_root = tmp_path / "candidate"
+    _write_candidate_receipt(source_root)
+    monkeypatch.setattr(
+        service,
+        "_generate_embedding",
+        lambda _text: (_ for _ in ()).throw(AssertionError("model call forbidden")),
+    )
+    monkeypatch.setattr(
+        service,
+        "_run_pgvector_probe",
+        lambda *_args: (_ for _ in ()).throw(AssertionError("DB call forbidden")),
+    )
+    payload = service.run_internal_rag_candidate_canary(
+        candidate_knowledge_receipt_root=source_root,
+    )
+
+    assert payload["status"] == "ready_for_canary"
+    assert payload["summary"]["ready_count"] == 1
+    assert payload["summary"]["executed_count"] == 0
+    assert payload["controlled_apply"]["model_call"] is False
+    assert payload["controlled_apply"]["database_write"] is False
+
+
+def test_internal_rag_canary_executes_read_only_pgvector_probe(tmp_path, monkeypatch):
+    from services import internal_rag_candidate_canary_service as service
+
+    source_root = tmp_path / "candidate"
+    output_root = tmp_path / "canary"
+    _write_candidate_receipt(source_root)
+    monkeypatch.setattr(
+        service, "_generate_embedding", lambda _text: [0.01] * service.RAG_EMBED_DIM
+    )
+    monkeypatch.setattr(
+        service,
+        "_verify_embedding_consistency",
+        lambda: {
+            "ok": True,
+            "reachable": ["gcp_ollama", "ollama_secondary"],
+            "max_diff": 0.0,
+            "errors": [],
+        },
+    )
+    monkeypatch.setattr(
+        service,
+        "_run_pgvector_probe",
+        lambda *_args: {
+            "transaction_read_only": True,
+            "exact_similarity": 1.0,
+            "probe_similarity": 0.91,
+            "ai_insights_table_present": True,
+            "database_write_performed": False,
+        },
+    )
+    payload = service.run_internal_rag_candidate_canary(
+        candidate_knowledge_receipt_root=source_root,
+        output_root=output_root,
+        execute=True,
+        write_receipt=True,
+    )
+
+    assert payload["summary"]["canary_passed_count"] == 1
+    assert payload["status"] in {"canary_passed_activation_blocked", "complete"}
+    item = payload["executed_items"][0]
+    assert item["canary_passed"] is True
+    assert item["transaction_read_only"] is True
+    assert item["writes_database"] is False
+    assert item["writes_ai_insights"] is False
+    assert item["writes_price_tables"] is False
+    assert payload["latest_execution"]["canary_passed"] is True
+    assert payload["run_identity"]["work_item_id"] == "RAG-P0-001"
+    assert item["run_identity"] == payload["run_identity"]
+
+
+def test_internal_rag_canary_requires_both_primary_gcp_hosts(tmp_path, monkeypatch):
+    from services import internal_rag_candidate_canary_service as service
+
+    source_root = tmp_path / "candidate"
+    _write_candidate_receipt(source_root)
+    monkeypatch.setattr(
+        service, "_generate_embedding", lambda _text: [0.01] * service.RAG_EMBED_DIM
+    )
+    monkeypatch.setattr(
+        service,
+        "_verify_embedding_consistency",
+        lambda: {
+            "ok": True,
+            "reachable": ["gcp_ollama", "ollama_111"],
+            "max_diff": 0.0,
+            "errors": ["ollama_secondary unavailable"],
+        },
+    )
+    monkeypatch.setattr(
+        service,
+        "_run_pgvector_probe",
+        lambda *_args: {
+            "transaction_read_only": True,
+            "exact_similarity": 1.0,
+            "probe_similarity": 0.91,
+            "ai_insights_table_present": True,
+            "database_write_performed": False,
+        },
+    )
+
+    payload = service.run_internal_rag_candidate_canary(
+        candidate_knowledge_receipt_root=source_root,
+        execute=True,
+    )
+
+    assert payload["status"] == "canary_failed"
+    item = payload["executed_items"][0]
+    assert item["canary_checks"]["cross_host_embedding_consistent"] is False
+    assert item["required_consistency_hosts"] == ["gcp_ollama", "ollama_secondary"]
+
+
+def test_internal_rag_canary_blocks_embedding_signature_drift(tmp_path):
+    from services.internal_rag_candidate_canary_service import run_internal_rag_candidate_canary
+
+    source_root = tmp_path / "candidate"
+    _write_candidate_receipt(source_root, signature="deadbeef0000")
+    payload = run_internal_rag_candidate_canary(
+        candidate_knowledge_receipt_root=source_root,
+    )
+
+    assert payload["status"] == "blocked"
+    assert payload["summary"]["ready_count"] == 0
+    assert payload["source_items"][0]["source_checks"][
+        "embedding_signature_matches_runtime"
+    ] is False
+
+
+def test_internal_rag_canary_cli_outputs_machine_readable_dry_run(tmp_path):
+    source_root = tmp_path / "candidate"
+    _write_candidate_receipt(source_root)
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "scripts/ops/run_internal_rag_candidate_canary.py",
+            "--candidate-knowledge-receipt-root",
+            str(source_root),
+        ],
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+    assert completed.returncode == 0
+    payload = json.loads(completed.stdout)
+    assert payload["status"] == "ready_for_canary"
+    assert payload["controlled_apply"]["database_write"] is False
+
+
+def test_internal_rag_canary_route_returns_readback(tmp_path, monkeypatch):
+    from flask import Flask
+    from routes import system_public_routes as routes
+    from services import internal_rag_candidate_canary_service as service
+
+    source_root = tmp_path / "candidate"
+    _write_candidate_receipt(source_root, platform="coupang_tw")
+    monkeypatch.setattr(
+        service, "DEFAULT_CANDIDATE_KNOWLEDGE_RECEIPT_ROOT", str(source_root)
+    )
+    app = Flask(__name__)
+    with app.test_request_context(
+        "/api/ai-automation/internal-rag-candidate-canary?platform=coupang_tw&execute=true"
+    ):
+        response = routes.ai_automation_internal_rag_candidate_canary_api.__wrapped__()
+    payload = response.get_json()
+
+    assert payload["policy"] == "controlled_internal_rag_candidate_canary_v1"
+    assert payload["summary"]["ready_count"] == 1
+    assert payload["execute"] is False
+
+
+def test_internal_rag_canary_http_route_is_read_only():
+    from routes import system_public_routes as routes
+
+    source = inspect.getsource(routes.ai_automation_internal_rag_candidate_canary_api)
+    assert "execute=False" in source
+    assert "write_receipt=False" in source
+    assert "get_current_user" not in source

tests/test_run_scheduler_embed_consistency.py

@@ -215,6 +215,7 @@ def test_v2_cron_blind_spot_list_has_failure_notifications(monkeypatch):
         "run_ppt_vision_audit",
         "run_embed_consistency_check",
         "run_ollama_111_usage_guard_check",
+        "run_internal_rag_candidate_canary_task",
     ]:
         source = inspect.getsource(getattr(run_scheduler, fn_name))
         assert "_notify_scheduler_failure(" in source
@@ -236,6 +237,50 @@ def test_roi_ai_smoke_and_daily_report_schedules_stay_staggered():
     assert "start_revenue_automation_watchdog()" in source
     assert "schedule.every(6).hours.do(run_action_plan_hygiene_task)" in source
     assert "schedule.every(15).minutes.do(run_ollama_111_usage_guard_check)" in source
+    assert 'schedule.every().day.at("04:45").do(run_internal_rag_candidate_canary_task)' in source
+
+
+def test_scheduled_internal_rag_canary_is_bounded_and_acknowledged(monkeypatch):
+    run_scheduler = _load_run_scheduler(monkeypatch)
+    import services.internal_rag_candidate_canary_service as canary_service
+    import services.telegram_templates as telegram_templates
+
+    calls = []
+    saved = []
+    monkeypatch.setattr(
+        canary_service,
+        "run_internal_rag_candidate_canary",
+        lambda **kwargs: calls.append(kwargs) or {
+            "success": True,
+            "status": "canary_passed_activation_blocked",
+            "run_identity": {
+                "trace_id": "trace-test",
+                "run_id": "run-test",
+                "work_item_id": "RAG-P0-001",
+            },
+            "summary": {
+                "source_receipt_count": 1,
+                "executed_count": 1,
+                "canary_passed_count": 1,
+            },
+            "closure_receipt": {},
+            "next_machine_action": "pin_embedding_model_then_enable_rag_controlled_canary",
+        },
+    )
+    monkeypatch.setattr(
+        telegram_templates,
+        "send_telegram_with_result",
+        lambda *_args, **_kwargs: {"ok": True, "sent": 1, "failed": 0},
+    )
+    monkeypatch.setattr(run_scheduler, "_save_stats", lambda name, data: saved.append((name, data)))
+
+    payload = run_scheduler.run_internal_rag_candidate_canary_task()
+
+    assert calls == [{"limit": 1, "execute": True, "write_receipt": True}]
+    assert payload["terminal_status"] == "verified_with_activation_blocker"
+    assert payload["acknowledgements"]["telegram"]["status"] == "acknowledged"
+    assert payload["acknowledgements"]["rag"] == "rag_canary_receipt_written"
+    assert saved[0][0] == "internal_rag_candidate_canary"
 
 
 def test_pchome_growth_backfill_catchup_uses_runtime_receipt_cooldown(monkeypatch, tmp_path):