Files
awoooi/apps/api/src/services/mcp_federation_service.py

1298 lines
51 KiB
Python

"""Durable read-only federation receipts for EwoooC and MOMO MCP/RAG runtime.
The source is a single committed production endpoint. This module deliberately
does not accept operator-provided URLs, redirects, credentials, raw tool output,
request bodies, response bodies, or RAG content. Only a strictly validated and
recomputed aggregate receipt may cross the federation boundary.
"""
from __future__ import annotations
import hashlib
import json
import re
import uuid
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any, Protocol
from urllib.parse import urlparse
import httpx
import structlog
from sqlalchemy import text
from src.core.config import settings
from src.core.http_client import get_general_client
from src.db.base import get_db_context
logger = structlog.get_logger(__name__)
PROJECT_ID = "awoooi"
SCHEMA_VERSION = "mcp_federation_runtime_v1"
SOURCE_SCHEMA_VERSION = "ewoooc_mcp_federation_receipt_v1"
SOURCE_POLICY = "public_aggregate_read_only_mcp_federation_v1"
SOURCE_ID = "ewoooc-momo-production"
SOURCE_URL = "https://mo.wooo.work/api/public/mcp-federation-readback"
_SOURCE_HOST = "mo.wooo.work"
_SOURCE_PATH = "/api/public/mcp-federation-readback"
_MAX_SOURCE_BYTES = 262_144
_MAX_SOURCE_AGE = timedelta(minutes=10)
_MAX_CLOCK_SKEW = timedelta(minutes=1)
_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
_SAFE_SLUG_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_.:-]{0,127}$")
_VERSION_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$")
_EXPECTED_PRODUCTS = {
"ewoooc": {
"canonical_repo": "wooo/ewoooc",
"runtime_role": "mcp_rag_automation_source_plane",
},
"momo-pro-system": {
"canonical_repo": "wooo/momo-pro-system",
"runtime_role": "commerce_runtime_consumer_plane",
},
}
_EXPECTED_OPERATION_BOUNDARIES = {
"read_only_observation_allowed": True,
"network_call_allowed": False,
"db_write_allowed": False,
"secret_read_allowed": False,
"external_tool_call_allowed": False,
"rag_write_allowed": False,
"production_price_write_allowed": False,
"request_or_response_body_storage_allowed": False,
}
_EXPECTED_DATA_BOUNDARIES = {
"scope": "aggregate_runtime_metadata_only",
"pixelrag": "public_visual_receipts_no_formal_product_write",
"rag": "pgvector_internal_only_no_public_content",
"mcp": "server_ids_and_counts_only_no_endpoint_or_payload",
}
_EXPECTED_ROOT_KEYS = {
"success",
"schema_version",
"policy",
"generated_at",
"status",
"receipt_count",
"receipts",
"blockers",
"operation_boundaries",
"next_machine_action",
}
_EXPECTED_RECEIPT_KEYS = {
"receipt_schema_version",
"product_id",
"canonical_repo",
"runtime_role",
"runtime_version",
"health_status",
"runtime",
"data_boundaries",
"operation_boundaries",
"blockers",
"observed_at",
"receipt_id",
"integrity",
}
_FORBIDDEN_KEYS = {
"authorization",
"cookie",
"credentials",
"database_url",
"endpoint",
"headers",
"password",
"private_key",
"raw_content",
"request_body",
"response_body",
"secret",
"server_url",
"token",
"tool_registry",
}
_FORBIDDEN_STRING_MARKERS = (
"http://",
"https://",
"postgresql://",
"postgresql+asyncpg://",
"redis://",
"bearer ",
"authorization:",
"-----begin ",
)
class FederationValidationError(ValueError):
"""Safe terminal for schema, integrity, freshness, or data-boundary failure."""
def __init__(self, code: str) -> None:
super().__init__(code)
self.code = code
@dataclass(frozen=True, slots=True)
class FederationFetchReceipt:
"""Bounded source observation; ``body`` exists in memory only."""
status: str
source_url: str
http_status: int | None = None
body: bytes = b""
error_class: str | None = None
FederationFetcher = Callable[[str, int], Awaitable[FederationFetchReceipt]]
class FederationRepository(Protocol):
"""Persistence boundary shared by PostgreSQL and focused unit-test fakes."""
async def start_run(self, payload: dict[str, Any]) -> None:
...
async def latest_receipts(self) -> dict[str, dict[str, Any]]:
...
async def save_receipt(self, payload: dict[str, Any]) -> None:
...
async def verify_run(
self,
run_id: uuid.UUID,
expected_fingerprints: dict[str, str],
) -> dict[str, Any]:
...
async def finish_run(self, payload: dict[str, Any]) -> None:
...
class PostgresFederationRepository:
"""Project-scoped PostgreSQL repository with independent readback verifier."""
async def start_run(self, payload: dict[str, Any]) -> None:
async with get_db_context(PROJECT_ID) as db:
await db.execute(
text(
"""
INSERT INTO awooop_mcp_federation_run (
run_id, trace_id, work_item_id, project_id, source_id,
trigger_kind, status, expected_product_count, receipt,
started_at
) VALUES (
:run_id, :trace_id, :work_item_id, :project_id,
:source_id, :trigger_kind, 'running',
:expected_product_count, CAST(:receipt AS jsonb),
:started_at
)
"""
),
{**payload, "receipt": _json(payload["receipt"])},
)
async def latest_receipts(self) -> dict[str, dict[str, Any]]:
async with get_db_context(PROJECT_ID) as db:
result = await db.execute(
text(
"""
SELECT DISTINCT ON (product_id)
product_id, normalized_fingerprint_sha256,
runtime_version, health_status, observed_at
FROM awooop_mcp_federated_runtime_receipt
WHERE project_id = :project_id
AND source_id = :source_id
AND verifier_status = 'verified'
ORDER BY product_id, received_at DESC
"""
),
{"project_id": PROJECT_ID, "source_id": SOURCE_ID},
)
return {
str(row["product_id"]): dict(row) for row in result.mappings().all()
}
async def save_receipt(self, payload: dict[str, Any]) -> None:
async with get_db_context(PROJECT_ID) as db:
await db.execute(
text(
"""
INSERT INTO awooop_mcp_federated_runtime_receipt (
run_id, trace_id, work_item_id, project_id, source_id,
product_id, canonical_repo, runtime_role,
runtime_version, health_status, change_state,
normalized_fingerprint_sha256, mcp_enabled,
mcp_server_count, mcp_server_ids, mcp_caller_count,
mcp_tool_count, rag_enabled,
rag_vector_store, rag_embedding_model,
rag_embedding_dim, rag_embedding_signature,
rag_embedding_version_state, pixelrag_enabled,
pixelrag_platform_count, pixelrag_platforms,
pixelrag_visual_rag_stage,
runtime_blockers, verifier_status, stage_receipts,
observed_at, received_at
) VALUES (
:run_id, :trace_id, :work_item_id, :project_id,
:source_id, :product_id, :canonical_repo, :runtime_role,
:runtime_version, :health_status, :change_state,
:normalized_fingerprint_sha256, :mcp_enabled,
:mcp_server_count, CAST(:mcp_server_ids AS jsonb),
:mcp_caller_count, :mcp_tool_count, :rag_enabled,
:rag_vector_store, :rag_embedding_model,
:rag_embedding_dim, :rag_embedding_signature,
:rag_embedding_version_state, :pixelrag_enabled,
:pixelrag_platform_count,
CAST(:pixelrag_platforms AS jsonb),
:pixelrag_visual_rag_stage,
CAST(:runtime_blockers AS jsonb), :verifier_status,
CAST(:stage_receipts AS jsonb), :observed_at,
:received_at
)
"""
),
{
**payload,
"mcp_server_ids": _json(payload["mcp_server_ids"]),
"pixelrag_platforms": _json(payload["pixelrag_platforms"]),
"runtime_blockers": _json(payload["runtime_blockers"]),
"stage_receipts": _json(payload["stage_receipts"]),
},
)
async def verify_run(
self,
run_id: uuid.UUID,
expected_fingerprints: dict[str, str],
) -> dict[str, Any]:
async with get_db_context(PROJECT_ID) as db:
result = await db.execute(
text(
"""
SELECT product_id, canonical_repo, runtime_role,
runtime_version, health_status,
normalized_fingerprint_sha256,
mcp_enabled, mcp_server_count, mcp_server_ids,
mcp_caller_count, mcp_tool_count,
rag_enabled, rag_vector_store, rag_embedding_model,
rag_embedding_dim, rag_embedding_signature,
rag_embedding_version_state,
pixelrag_enabled, pixelrag_platform_count,
pixelrag_platforms, pixelrag_visual_rag_stage,
runtime_blockers, verifier_status
FROM awooop_mcp_federated_runtime_receipt
WHERE project_id = :project_id AND run_id = :run_id
ORDER BY product_id
"""
),
{"project_id": PROJECT_ID, "run_id": run_id},
)
actual: dict[str, str] = {}
recomputed_match = True
for row in result.mappings().all():
if row["verifier_status"] != "verified":
continue
product_id = str(row["product_id"])
stored = str(row["normalized_fingerprint_sha256"])
recomputed = _recompute_persisted_fingerprint(dict(row))
recomputed_match = recomputed_match and stored == recomputed
actual[product_id] = stored
return {
"status": (
"verified"
if actual == expected_fingerprints and recomputed_match
else "failed"
),
"verified_product_count": len(actual),
"fingerprints_match": actual == expected_fingerprints,
"persisted_fields_recompute_match": recomputed_match,
}
async def finish_run(self, payload: dict[str, Any]) -> None:
async with get_db_context(PROJECT_ID) as db:
await db.execute(
text(
"""
UPDATE awooop_mcp_federation_run
SET status = :status,
received_product_count = :received_product_count,
verified_product_count = :verified_product_count,
runtime_blocked_product_count = :runtime_blocked_product_count,
error_count = :error_count,
error_class = :error_class,
receipt = CAST(:receipt AS jsonb),
ended_at = :ended_at
WHERE project_id = :project_id AND run_id = :run_id
"""
),
{**payload, "receipt": _json(payload["receipt"])},
)
async def fetch_federation_source(
source_url: str,
timeout_seconds: int,
) -> FederationFetchReceipt:
"""Fetch exactly one allowlisted JSON endpoint with redirects disabled."""
if not _is_allowed_source_url(source_url):
return FederationFetchReceipt(
status="policy_blocked",
source_url=source_url,
error_class="source_url_not_allowlisted",
)
client = await get_general_client()
try:
async with client.stream(
"GET",
source_url,
follow_redirects=False,
timeout=httpx.Timeout(float(timeout_seconds), connect=5.0),
headers={
"Accept": "application/json",
"User-Agent": "awoooi-mcp-federation-verifier/1.0",
},
) as response:
if response.status_code != 200:
error_class = (
"redirect_blocked"
if response.is_redirect
else f"http_{response.status_code}"
)
return FederationFetchReceipt(
status="policy_blocked" if response.is_redirect else "error",
source_url=source_url,
http_status=response.status_code,
error_class=error_class,
)
content_type = response.headers.get("content-type", "").lower()
if "application/json" not in content_type:
return FederationFetchReceipt(
status="error",
source_url=source_url,
http_status=response.status_code,
error_class="unsupported_content_type",
)
content_length = response.headers.get("content-length")
if content_length:
try:
if int(content_length) > _MAX_SOURCE_BYTES:
return FederationFetchReceipt(
status="error",
source_url=source_url,
http_status=response.status_code,
error_class="source_payload_too_large",
)
except ValueError:
return FederationFetchReceipt(
status="error",
source_url=source_url,
http_status=response.status_code,
error_class="invalid_content_length",
)
body = bytearray()
async for chunk in response.aiter_bytes():
body.extend(chunk)
if len(body) > _MAX_SOURCE_BYTES:
return FederationFetchReceipt(
status="error",
source_url=source_url,
http_status=response.status_code,
error_class="source_payload_too_large",
)
return FederationFetchReceipt(
status="ok",
source_url=source_url,
http_status=response.status_code,
body=bytes(body),
)
except httpx.TimeoutException:
return FederationFetchReceipt(
status="error",
source_url=source_url,
error_class="source_timeout",
)
except (httpx.RequestError, ValueError):
return FederationFetchReceipt(
status="error",
source_url=source_url,
error_class="source_network_error",
)
async def run_mcp_federation_once(
*,
trigger_kind: str = "scheduled",
repository: FederationRepository | None = None,
fetcher: FederationFetcher | None = None,
generated_at: datetime | None = None,
) -> dict[str, Any]:
"""Fetch, validate, normalize, persist, and independently verify one run."""
repository = repository or PostgresFederationRepository()
fetcher = fetcher or fetch_federation_source
now = generated_at or datetime.now(UTC)
if now.tzinfo is None:
now = now.replace(tzinfo=UTC)
run_id = uuid.uuid4()
trace_id = f"mcp-federation-{run_id}"
work_item_id = f"MCP-FEDERATION-{now.astimezone(UTC):%Y%m%dT%H%M%SZ}"
source_url_sha256 = hashlib.sha256(SOURCE_URL.encode("utf-8")).hexdigest()
start_receipt = {
"schema_version": SCHEMA_VERSION,
"source_id": SOURCE_ID,
"source_url_sha256": source_url_sha256,
"expected_product_ids": sorted(_EXPECTED_PRODUCTS),
"raw_source_payload_stored": False,
"source_redirect_allowed": False,
"credential_sent": False,
"external_tool_call_performed": False,
"external_rag_write_performed": False,
"production_mutation_performed": False,
"github_used": False,
}
await repository.start_run(
{
"run_id": run_id,
"trace_id": trace_id,
"work_item_id": work_item_id,
"project_id": PROJECT_ID,
"source_id": SOURCE_ID,
"trigger_kind": trigger_kind,
"expected_product_count": len(_EXPECTED_PRODUCTS),
"receipt": start_receipt,
"started_at": now,
}
)
counts = {
"received_product_count": 0,
"verified_product_count": 0,
"runtime_blocked_product_count": 0,
"error_count": 0,
}
error_class: str | None = None
normalized: list[dict[str, Any]] = []
try:
fetched = await fetcher(
SOURCE_URL,
settings.MCP_FEDERATION_SOURCE_TIMEOUT_SECONDS,
)
if fetched.status != "ok":
raise FederationValidationError(
fetched.error_class or "source_read_failed"
)
normalized = validate_federation_payload(fetched.body, received_at=now)
counts["received_product_count"] = len(normalized)
previous = await repository.latest_receipts()
expected_fingerprints: dict[str, str] = {}
for receipt in normalized:
product_id = str(receipt["product_id"])
previous_row = previous.get(product_id)
previous_fingerprint = (
str(previous_row.get("normalized_fingerprint_sha256"))
if previous_row
else None
)
change_state = (
"baseline_created"
if previous_fingerprint is None
else (
"no_change"
if previous_fingerprint
== receipt["normalized_fingerprint_sha256"]
else "runtime_changed"
)
)
receipt["change_state"] = change_state
row = build_federated_receipt_row(
receipt,
run_id=run_id,
trace_id=trace_id,
work_item_id=work_item_id,
change_state=change_state,
previous=previous_row,
received_at=now,
)
await repository.save_receipt(row)
expected_fingerprints[product_id] = str(
receipt["normalized_fingerprint_sha256"]
)
if receipt["blockers"]:
counts["runtime_blocked_product_count"] += 1
post_verifier = await repository.verify_run(run_id, expected_fingerprints)
if post_verifier.get("status") != "verified":
raise FederationValidationError("durable_post_verifier_failed")
counts["verified_product_count"] = int(
post_verifier["verified_product_count"]
)
status = (
"completed_verified_with_runtime_blockers"
if counts["runtime_blocked_product_count"]
else "completed_verified"
)
final_receipt = {
**start_receipt,
"status": status,
**counts,
"sensor_source_receipt": {
"status": "verified",
"http_status": fetched.http_status,
"body_stored": False,
},
"source_of_truth_diff": {
"changed_product_count": sum(
1
for receipt in normalized
if str(receipt.get("change_state")) == "runtime_changed"
),
},
"risk_policy_decision": {
"risk": "medium",
"decision": "bounded_normalized_receipt_write_only",
},
"check_mode": {
"status": "source_schema_integrity_and_freshness_verified",
"external_process_started": False,
},
"bounded_execution": {
"status": "normalized_rows_committed",
"row_count": counts["verified_product_count"],
"external_runtime_mutated": False,
},
"independent_post_verifier": post_verifier,
"rollback": {
"status": "no_external_write_terminal",
"schema_evidence_retained": True,
},
"learning_writeback": {
"status": "durable_control_plane_receipts_committed",
"rag_or_km_content_write_performed": False,
},
}
await repository.finish_run(
{
"run_id": run_id,
"project_id": PROJECT_ID,
"status": status,
**counts,
"error_class": None,
"receipt": final_receipt,
"ended_at": datetime.now(UTC),
}
)
return {
"schema_version": SCHEMA_VERSION,
"run_id": str(run_id),
"trace_id": trace_id,
"work_item_id": work_item_id,
"status": status,
**counts,
}
except FederationValidationError as exc:
counts["error_count"] = 1
error_class = exc.code
except Exception:
counts["error_count"] = 1
error_class = "controller_execution_error"
logger.exception(
"mcp_federation_controller_execution_failed",
run_id=str(run_id),
)
failure_receipt = {
**start_receipt,
"status": "partial_degraded",
**counts,
"error_class": error_class,
"independent_post_verifier": {
"status": "partial",
"external_runtime_mutated": False,
},
"rollback": {"status": "no_external_write_terminal"},
"learning_writeback": {
"status": "failure_receipt_committed",
"raw_source_payload_stored": False,
},
}
await repository.finish_run(
{
"run_id": run_id,
"project_id": PROJECT_ID,
"status": "partial_degraded",
**counts,
"error_class": error_class,
"receipt": failure_receipt,
"ended_at": datetime.now(UTC),
}
)
return {
"schema_version": SCHEMA_VERSION,
"run_id": str(run_id),
"trace_id": trace_id,
"work_item_id": work_item_id,
"status": "partial_degraded",
**counts,
"error_class": error_class,
}
def validate_federation_payload(
body: bytes,
*,
received_at: datetime,
) -> list[dict[str, Any]]:
"""Return exactly two safe normalized receipts or fail closed."""
if len(body) > _MAX_SOURCE_BYTES:
raise FederationValidationError("source_payload_too_large")
try:
payload = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
raise FederationValidationError("source_json_invalid") from None
if not isinstance(payload, dict):
raise FederationValidationError("source_schema_invalid")
_reject_sensitive_content(payload)
if set(payload) != _EXPECTED_ROOT_KEYS or payload.get("success") is not True:
raise FederationValidationError("source_schema_invalid")
if payload.get("schema_version") != SOURCE_SCHEMA_VERSION:
raise FederationValidationError("source_schema_version_invalid")
if payload.get("policy") != SOURCE_POLICY:
raise FederationValidationError("source_policy_invalid")
if payload.get("operation_boundaries") != _EXPECTED_OPERATION_BOUNDARIES:
raise FederationValidationError("source_operation_boundary_invalid")
root_blockers = _safe_slugs(payload.get("blockers"), max_items=32)
expected_status = "partial_degraded" if root_blockers else "ready"
if payload.get("status") != expected_status:
raise FederationValidationError("source_status_blocker_mismatch")
expected_next_action = (
"pin_embedding_model_and_verify_runtime_receipts"
if root_blockers
else "continue_scheduled_read_only_federation"
)
if payload.get("next_machine_action") != expected_next_action:
raise FederationValidationError("source_next_action_invalid")
generated_at = _parse_fresh_timestamp(payload.get("generated_at"), received_at)
receipts = payload.get("receipts")
if not isinstance(receipts, list) or len(receipts) != len(_EXPECTED_PRODUCTS):
raise FederationValidationError("source_receipt_count_invalid")
if payload.get("receipt_count") != len(receipts):
raise FederationValidationError("source_receipt_count_mismatch")
normalized: list[dict[str, Any]] = []
seen: set[str] = set()
for source_receipt in receipts:
if not isinstance(source_receipt, dict):
raise FederationValidationError("source_receipt_schema_invalid")
if set(source_receipt) != _EXPECTED_RECEIPT_KEYS:
raise FederationValidationError("source_receipt_schema_invalid")
product_id = _bounded_string(source_receipt.get("product_id"), 64)
if product_id not in _EXPECTED_PRODUCTS or product_id in seen:
raise FederationValidationError("source_product_identity_invalid")
seen.add(product_id)
expected = _EXPECTED_PRODUCTS[product_id]
if source_receipt.get("receipt_schema_version") != SOURCE_SCHEMA_VERSION:
raise FederationValidationError("source_receipt_schema_version_invalid")
if source_receipt.get("canonical_repo") != expected["canonical_repo"]:
raise FederationValidationError("source_canonical_repo_invalid")
if source_receipt.get("runtime_role") != expected["runtime_role"]:
raise FederationValidationError("source_runtime_role_invalid")
if source_receipt.get("operation_boundaries") != _EXPECTED_OPERATION_BOUNDARIES:
raise FederationValidationError("source_receipt_boundary_invalid")
if source_receipt.get("data_boundaries") != _EXPECTED_DATA_BOUNDARIES:
raise FederationValidationError("source_data_boundary_invalid")
observed_at = _parse_fresh_timestamp(
source_receipt.get("observed_at"), received_at
)
if abs(observed_at - generated_at) > _MAX_CLOCK_SKEW:
raise FederationValidationError("source_timestamp_mismatch")
runtime_version = _bounded_string(source_receipt.get("runtime_version"), 64)
if not _VERSION_PATTERN.fullmatch(runtime_version):
raise FederationValidationError("source_runtime_version_invalid")
health_status = source_receipt.get("health_status")
if health_status not in {"ready", "partial_degraded"}:
raise FederationValidationError("source_health_status_invalid")
blockers = _safe_slugs(source_receipt.get("blockers"), max_items=32)
if blockers != root_blockers:
raise FederationValidationError("source_product_blocker_mismatch")
if health_status == "ready" and blockers:
raise FederationValidationError("source_health_blocker_mismatch")
if health_status == "partial_degraded" and not blockers:
raise FederationValidationError("source_health_blocker_mismatch")
runtime = source_receipt.get("runtime")
if not isinstance(runtime, dict) or set(runtime) != {"mcp", "rag", "pixelrag"}:
raise FederationValidationError("source_runtime_schema_invalid")
mcp = _normalize_mcp(runtime.get("mcp"))
rag = _normalize_rag(runtime.get("rag"))
pixelrag = _normalize_pixelrag(runtime.get("pixelrag"))
fingerprint = _verify_source_fingerprint(source_receipt)
normalized.append(
{
"product_id": product_id,
"canonical_repo": expected["canonical_repo"],
"runtime_role": expected["runtime_role"],
"runtime_version": runtime_version,
"health_status": health_status,
"mcp": mcp,
"rag": rag,
"pixelrag": pixelrag,
"blockers": blockers,
"observed_at": observed_at,
"normalized_fingerprint_sha256": fingerprint,
}
)
if seen != set(_EXPECTED_PRODUCTS):
raise FederationValidationError("source_product_set_invalid")
return sorted(normalized, key=lambda item: str(item["product_id"]))
def build_federated_receipt_row(
receipt: dict[str, Any],
*,
run_id: uuid.UUID,
trace_id: str,
work_item_id: str,
change_state: str,
previous: dict[str, Any] | None,
received_at: datetime,
) -> dict[str, Any]:
"""Map one validated receipt into the explicit no-raw-content DB contract."""
mcp = receipt["mcp"]
rag = receipt["rag"]
pixelrag = receipt["pixelrag"]
stage_receipts = {
"sensor_source_receipt": {
"status": "verified",
"source_id": SOURCE_ID,
"source_url_sha256": hashlib.sha256(
SOURCE_URL.encode("utf-8")
).hexdigest(),
"raw_payload_stored": False,
},
"normalized_asset_identity": {
"product_id": receipt["product_id"],
"canonical_repo": receipt["canonical_repo"],
"runtime_role": receipt["runtime_role"],
},
"source_of_truth_diff": {
"change_state": change_state,
"previous_runtime_version": (
str(previous.get("runtime_version")) if previous else None
),
"observed_runtime_version": receipt["runtime_version"],
"previous_health_status": (
str(previous.get("health_status")) if previous else None
),
"observed_health_status": receipt["health_status"],
},
"decision": {
"candidate_action": "persist_normalized_read_only_receipt",
"runtime_promotion_allowed": False,
},
"risk_policy": {
"risk": "medium",
"bounded_write": "awoooi_control_plane_receipt_only",
},
"check_mode": {
"status": "schema_integrity_freshness_and_data_boundary_verified",
"external_process_started": False,
},
"bounded_execution": {
"status": "one_idempotent_product_receipt_insert",
"external_runtime_mutated": False,
},
"post_verifier_and_rollback": {
"status": "pending_run_level_durable_verifier",
"rollback": "no_external_write_terminal",
},
"learning_writeback": {
"status": "same_row_commit",
"rag_or_km_content_write_performed": False,
},
}
return {
"run_id": run_id,
"trace_id": trace_id,
"work_item_id": work_item_id,
"project_id": PROJECT_ID,
"source_id": SOURCE_ID,
"product_id": receipt["product_id"],
"canonical_repo": receipt["canonical_repo"],
"runtime_role": receipt["runtime_role"],
"runtime_version": receipt["runtime_version"],
"health_status": receipt["health_status"],
"change_state": change_state,
"normalized_fingerprint_sha256": receipt[
"normalized_fingerprint_sha256"
],
"mcp_enabled": mcp["enabled"],
"mcp_server_count": mcp["server_count"],
"mcp_server_ids": mcp["server_ids"],
"mcp_caller_count": mcp["caller_count"],
"mcp_tool_count": mcp["tool_count"],
"rag_enabled": rag["enabled"],
"rag_vector_store": rag["vector_store"],
"rag_embedding_model": rag["embedding_model"],
"rag_embedding_dim": rag["embedding_dim"],
"rag_embedding_signature": rag["embedding_signature"],
"rag_embedding_version_state": rag["embedding_version_state"],
"pixelrag_enabled": pixelrag["enabled"],
"pixelrag_platform_count": pixelrag["platform_count"],
"pixelrag_platforms": pixelrag["platforms"],
"pixelrag_visual_rag_stage": pixelrag["visual_rag_stage"],
"runtime_blockers": receipt["blockers"],
"verifier_status": "verified",
"stage_receipts": stage_receipts,
"observed_at": receipt["observed_at"],
"received_at": received_at,
}
async def collect_mcp_federation_readback() -> dict[str, Any]:
"""Return the newest run plus latest fresh verified product receipts."""
try:
async with get_db_context(PROJECT_ID) as db:
run_result = await db.execute(
text(
"""
SELECT run_id, trace_id, work_item_id, trigger_kind, status,
expected_product_count, received_product_count,
verified_product_count, runtime_blocked_product_count,
error_count, error_class, started_at, ended_at,
started_at >= NOW() - make_interval(
secs => :freshness_seconds
) AS fresh
FROM awooop_mcp_federation_run
WHERE project_id = :project_id AND source_id = :source_id
ORDER BY started_at DESC
LIMIT 1
"""
),
{
"project_id": PROJECT_ID,
"source_id": SOURCE_ID,
"freshness_seconds": settings.MCP_FEDERATION_INTERVAL_SECONDS * 2,
},
)
latest = run_result.mappings().one_or_none()
if latest is None:
return _pending_readback()
receipt_result = await db.execute(
text(
"""
SELECT DISTINCT ON (product_id)
product_id, canonical_repo, runtime_role,
runtime_version, health_status, change_state,
mcp_enabled, mcp_server_count, mcp_server_ids,
mcp_caller_count, mcp_tool_count,
rag_enabled, rag_vector_store, rag_embedding_model,
rag_embedding_dim, rag_embedding_version_state,
pixelrag_enabled, pixelrag_platform_count,
pixelrag_platforms,
pixelrag_visual_rag_stage, runtime_blockers,
observed_at, received_at,
received_at >= NOW() - make_interval(
secs => :freshness_seconds
) AS fresh
FROM awooop_mcp_federated_runtime_receipt
WHERE project_id = :project_id
AND source_id = :source_id
AND verifier_status = 'verified'
ORDER BY product_id, received_at DESC
"""
),
{
"project_id": PROJECT_ID,
"source_id": SOURCE_ID,
"freshness_seconds": settings.MCP_FEDERATION_INTERVAL_SECONDS * 2,
},
)
receipts = [
{
"product_id": str(row["product_id"]),
"canonical_repo": str(row["canonical_repo"]),
"runtime_role": str(row["runtime_role"]),
"runtime_version": str(row["runtime_version"]),
"health_status": str(row["health_status"]),
"change_state": str(row["change_state"]),
"mcp": {
"enabled": bool(row["mcp_enabled"]),
"server_count": int(row["mcp_server_count"]),
"server_ids": _json_string_list(row["mcp_server_ids"]),
"caller_count": int(row["mcp_caller_count"]),
"tool_count": int(row["mcp_tool_count"]),
},
"rag": {
"enabled": bool(row["rag_enabled"]),
"vector_store": str(row["rag_vector_store"]),
"embedding_model": str(row["rag_embedding_model"]),
"embedding_dim": int(row["rag_embedding_dim"]),
"embedding_version_state": str(
row["rag_embedding_version_state"]
),
},
"pixelrag": {
"enabled": bool(row["pixelrag_enabled"]),
"platform_count": int(row["pixelrag_platform_count"]),
"platforms": _json_string_list(row["pixelrag_platforms"]),
"visual_rag_stage": str(row["pixelrag_visual_rag_stage"]),
},
"blockers": _json_string_list(row["runtime_blockers"]),
"observed_at": _iso(row["observed_at"]),
"received_at": _iso(row["received_at"]),
"fresh": bool(row["fresh"]),
}
for row in receipt_result.mappings().all()
]
latest_run = {
"run_id": str(latest["run_id"]),
"trace_id": str(latest["trace_id"]),
"work_item_id": str(latest["work_item_id"]),
"trigger_kind": str(latest["trigger_kind"]),
"status": str(latest["status"]),
"expected_product_count": int(latest["expected_product_count"]),
"received_product_count": int(latest["received_product_count"]),
"verified_product_count": int(latest["verified_product_count"]),
"runtime_blocked_product_count": int(
latest["runtime_blocked_product_count"]
),
"error_count": int(latest["error_count"]),
"error_class": latest["error_class"],
"started_at": _iso(latest["started_at"]),
"ended_at": _iso(latest["ended_at"]),
"fresh": bool(latest["fresh"]),
}
blockers: list[str] = []
if not latest_run["fresh"]:
blockers.append("federation_run_stale")
if latest_run["status"] == "partial_degraded":
blockers.append("federation_latest_run_partial_degraded")
if len([row for row in receipts if row["fresh"]]) != len(_EXPECTED_PRODUCTS):
blockers.append("federated_product_runtime_receipts_incomplete")
return {
"status": "ready" if not blockers else "degraded",
"controller_configured": True,
"controller_owner": "signal_worker",
"source_id": SOURCE_ID,
"source_url_sha256": hashlib.sha256(
SOURCE_URL.encode("utf-8")
).hexdigest(),
"interval_seconds": settings.MCP_FEDERATION_INTERVAL_SECONDS,
"latest_run": latest_run,
"receipts": receipts,
"verified_fresh_receipt_count": len(
[row for row in receipts if row["fresh"]]
),
"expected_receipt_count": len(_EXPECTED_PRODUCTS),
"blockers": blockers,
"operation_boundaries": {
"external_runtime_mutation_allowed": False,
"external_rag_write_allowed": False,
"raw_source_payload_stored": False,
"credential_sent": False,
},
}
except Exception as exc:
logger.warning(
"mcp_federation_runtime_readback_failed",
error_type=type(exc).__name__,
)
return {
**_pending_readback(),
"status": "degraded",
"blockers": ["federation_runtime_readback_failed"],
}
def _normalize_mcp(value: Any) -> dict[str, Any]:
if not isinstance(value, dict) or set(value) != {
"enabled",
"server_count",
"server_ids",
"caller_count",
"tool_count",
}:
raise FederationValidationError("source_mcp_schema_invalid")
server_ids = _safe_slugs(value.get("server_ids"), max_items=32)
server_count = _bounded_count(value.get("server_count"), maximum=32)
if server_count != len(server_ids):
raise FederationValidationError("source_mcp_server_count_mismatch")
return {
"enabled": _strict_bool(value.get("enabled")),
"server_count": server_count,
"server_ids": server_ids,
"caller_count": _bounded_count(value.get("caller_count"), maximum=64),
"tool_count": _bounded_count(value.get("tool_count"), maximum=1_000),
}
def _normalize_rag(value: Any) -> dict[str, Any]:
if not isinstance(value, dict) or set(value) != {
"enabled",
"vector_store",
"embedding_model",
"embedding_dim",
"embedding_signature",
"embedding_version_state",
}:
raise FederationValidationError("source_rag_schema_invalid")
vector_store = _bounded_string(value.get("vector_store"), 32)
if vector_store != "pgvector":
raise FederationValidationError("source_rag_vector_store_invalid")
embedding_state = _bounded_string(value.get("embedding_version_state"), 32)
if embedding_state not in {"floating_tag_detected", "explicit_or_unknown"}:
raise FederationValidationError("source_rag_version_state_invalid")
return {
"enabled": _strict_bool(value.get("enabled")),
"vector_store": vector_store,
"embedding_model": _bounded_string(value.get("embedding_model"), 160),
"embedding_dim": _bounded_count(value.get("embedding_dim"), maximum=65_536),
"embedding_signature": _bounded_string(
value.get("embedding_signature"), 256, allow_empty=True
),
"embedding_version_state": embedding_state,
}
def _normalize_pixelrag(value: Any) -> dict[str, Any]:
if not isinstance(value, dict) or set(value) != {
"enabled",
"platform_count",
"platforms",
"visual_rag_stage",
"formal_product_write_allowed",
}:
raise FederationValidationError("source_pixelrag_schema_invalid")
platforms = _safe_slugs(value.get("platforms"), max_items=32)
platform_count = _bounded_count(value.get("platform_count"), maximum=32)
if platform_count != len(platforms):
raise FederationValidationError("source_pixelrag_platform_count_mismatch")
if value.get("formal_product_write_allowed") is not False:
raise FederationValidationError("source_pixelrag_write_boundary_invalid")
return {
"enabled": _strict_bool(value.get("enabled")),
"platform_count": platform_count,
"platforms": platforms,
"visual_rag_stage": _bounded_string(value.get("visual_rag_stage"), 64),
"formal_product_write_allowed": False,
}
def _verify_source_fingerprint(receipt: dict[str, Any]) -> str:
integrity = receipt.get("integrity")
if (
not isinstance(integrity, dict)
or set(integrity)
!= {
"algorithm",
"canonicalization",
"excluded_fields",
"normalized_fingerprint_sha256",
}
or integrity.get("algorithm") != "sha256"
):
raise FederationValidationError("source_integrity_schema_invalid")
if integrity.get("canonicalization") != "json_sort_keys_compact_v1":
raise FederationValidationError("source_integrity_canonicalization_invalid")
if integrity.get("excluded_fields") != ["receipt_id", "observed_at", "integrity"]:
raise FederationValidationError("source_integrity_scope_invalid")
claimed = integrity.get("normalized_fingerprint_sha256")
if not isinstance(claimed, str) or not _SHA256_PATTERN.fullmatch(claimed):
raise FederationValidationError("source_integrity_hash_invalid")
covered = {
key: value
for key, value in receipt.items()
if key not in {"receipt_id", "observed_at", "integrity"}
}
canonical = json.dumps(
covered,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
actual = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
if actual != claimed:
raise FederationValidationError("source_integrity_mismatch")
expected_receipt_id = f"mcp-federation:{receipt.get('product_id')}:{actual[:16]}"
if receipt.get("receipt_id") != expected_receipt_id:
raise FederationValidationError("source_receipt_id_invalid")
return actual
def _recompute_persisted_fingerprint(row: dict[str, Any]) -> str:
"""Rebuild the source hash scope from persisted normalized columns only."""
covered = {
"receipt_schema_version": SOURCE_SCHEMA_VERSION,
"product_id": str(row["product_id"]),
"canonical_repo": str(row["canonical_repo"]),
"runtime_role": str(row["runtime_role"]),
"runtime_version": str(row["runtime_version"]),
"health_status": str(row["health_status"]),
"runtime": {
"mcp": {
"enabled": bool(row["mcp_enabled"]),
"server_count": int(row["mcp_server_count"]),
"server_ids": _json_string_list(row["mcp_server_ids"]),
"caller_count": int(row["mcp_caller_count"]),
"tool_count": int(row["mcp_tool_count"]),
},
"rag": {
"enabled": bool(row["rag_enabled"]),
"vector_store": str(row["rag_vector_store"]),
"embedding_model": str(row["rag_embedding_model"]),
"embedding_dim": int(row["rag_embedding_dim"]),
"embedding_signature": str(row["rag_embedding_signature"]),
"embedding_version_state": str(
row["rag_embedding_version_state"]
),
},
"pixelrag": {
"enabled": bool(row["pixelrag_enabled"]),
"platform_count": int(row["pixelrag_platform_count"]),
"platforms": _json_string_list(row["pixelrag_platforms"]),
"visual_rag_stage": str(row["pixelrag_visual_rag_stage"]),
"formal_product_write_allowed": False,
},
},
"data_boundaries": _EXPECTED_DATA_BOUNDARIES,
"operation_boundaries": _EXPECTED_OPERATION_BOUNDARIES,
"blockers": _json_string_list(row["runtime_blockers"]),
}
canonical = json.dumps(
covered,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _reject_sensitive_content(value: Any) -> None:
if isinstance(value, dict):
for key, nested in value.items():
if str(key).strip().lower() in _FORBIDDEN_KEYS:
raise FederationValidationError("source_sensitive_field_detected")
_reject_sensitive_content(nested)
return
if isinstance(value, list):
for nested in value:
_reject_sensitive_content(nested)
return
if isinstance(value, str):
lowered = value.lower()
if any(marker in lowered for marker in _FORBIDDEN_STRING_MARKERS):
raise FederationValidationError("source_sensitive_value_detected")
def _parse_fresh_timestamp(value: Any, received_at: datetime) -> datetime:
if not isinstance(value, str) or len(value) > 64:
raise FederationValidationError("source_timestamp_invalid")
try:
parsed = datetime.fromisoformat(value)
except ValueError:
raise FederationValidationError("source_timestamp_invalid") from None
if parsed.tzinfo is None:
raise FederationValidationError("source_timestamp_timezone_missing")
received = received_at if received_at.tzinfo else received_at.replace(tzinfo=UTC)
if parsed > received + _MAX_CLOCK_SKEW:
raise FederationValidationError("source_timestamp_in_future")
if received - parsed > _MAX_SOURCE_AGE:
raise FederationValidationError("source_timestamp_stale")
return parsed
def _is_allowed_source_url(value: str) -> bool:
try:
parsed = urlparse(value)
return (
parsed.scheme == "https"
and parsed.hostname == _SOURCE_HOST
and parsed.port in {None, 443}
and parsed.path == _SOURCE_PATH
and not parsed.username
and not parsed.password
and not parsed.params
and not parsed.query
and not parsed.fragment
)
except ValueError:
return False
def _strict_bool(value: Any) -> bool:
if not isinstance(value, bool):
raise FederationValidationError("source_boolean_invalid")
return bool(value)
def _bounded_count(value: Any, *, maximum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 0 or value > maximum:
raise FederationValidationError("source_count_invalid")
return int(value)
def _bounded_string(value: Any, maximum: int, *, allow_empty: bool = False) -> str:
if not isinstance(value, str) or len(value) > maximum:
raise FederationValidationError("source_string_invalid")
if not allow_empty and not value:
raise FederationValidationError("source_string_invalid")
return value
def _safe_slugs(value: Any, *, max_items: int) -> list[str]:
if not isinstance(value, list) or len(value) > max_items:
raise FederationValidationError("source_slug_list_invalid")
rows: list[str] = []
for item in value:
if not isinstance(item, str) or not _SAFE_SLUG_PATTERN.fullmatch(item):
raise FederationValidationError("source_slug_invalid")
rows.append(item)
if rows != sorted(set(rows)):
raise FederationValidationError("source_slug_list_not_canonical")
return rows
def _pending_readback() -> dict[str, Any]:
return {
"status": "pending_first_run",
"controller_configured": True,
"controller_owner": "signal_worker",
"source_id": SOURCE_ID,
"source_url_sha256": hashlib.sha256(SOURCE_URL.encode("utf-8")).hexdigest(),
"interval_seconds": settings.MCP_FEDERATION_INTERVAL_SECONDS,
"latest_run": None,
"receipts": [],
"verified_fresh_receipt_count": 0,
"expected_receipt_count": len(_EXPECTED_PRODUCTS),
"blockers": ["first_federation_runtime_receipt_pending"],
"operation_boundaries": {
"external_runtime_mutation_allowed": False,
"external_rag_write_allowed": False,
"raw_source_payload_stored": False,
"credential_sent": False,
},
}
def _json_string_list(value: Any) -> list[str]:
if isinstance(value, list):
return [str(item) for item in value]
if isinstance(value, str):
try:
decoded = json.loads(value)
except json.JSONDecodeError:
return []
return [str(item) for item in decoded] if isinstance(decoded, list) else []
return []
def _iso(value: Any) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _json(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)