936 lines
34 KiB
Python
936 lines
34 KiB
Python
"""Durable, fail-closed lifecycle receipts for external MCP candidates.
|
|
|
|
The controller reads only committed Smithery and mcp.so catalog URLs. It stores
|
|
normalized metadata and hashes, never raw external content. A version/content
|
|
change cannot reach replay, canary, rollout, or promotion unless immutable
|
|
supply-chain receipts and a registered internal replay adapter are present.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import html
|
|
import json
|
|
import re
|
|
import uuid
|
|
from collections.abc import Awaitable, Callable
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from html.parser import HTMLParser
|
|
from typing import Any, Protocol
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
import httpx
|
|
import structlog
|
|
from sqlalchemy import text
|
|
|
|
from src.core.config import settings
|
|
from src.core.http_client import get_general_client
|
|
from src.db.base import get_db_context
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
PROJECT_ID = "awoooi"
|
|
SCHEMA_VERSION = "mcp_version_lifecycle_runtime_v1"
|
|
_MAX_SOURCE_BYTES = 2_000_000
|
|
_MAX_REDIRECTS = 2
|
|
_MAX_CONCURRENT_FETCHES = 3
|
|
_SOURCE_HOSTS = {
|
|
"mcp-so": frozenset({"mcp.so", "www.mcp.so"}),
|
|
"smithery": frozenset({"smithery.ai", "www.smithery.ai"}),
|
|
}
|
|
_VERSION_PATTERNS = (
|
|
re.compile(
|
|
rb"""(?ix)["'](?:latestVersion|packageVersion|softwareVersion)["']\s*[:=]\s*["']v?"""
|
|
rb"""([0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?)["']"""
|
|
),
|
|
re.compile(
|
|
rb"""(?ix)data-(?:version|package-version)=["']v?"""
|
|
rb"""([0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?)["']"""
|
|
),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CatalogFetchReceipt:
|
|
"""Bounded source observation. The raw body is never persisted."""
|
|
|
|
status: str
|
|
source_url: str
|
|
http_status: int | None = None
|
|
body: bytes = b""
|
|
error_class: str | None = None
|
|
redirect_count: int = 0
|
|
|
|
|
|
CatalogFetcher = Callable[[str, str, int], Awaitable[CatalogFetchReceipt]]
|
|
|
|
|
|
class _StableMetadataParser(HTMLParser):
|
|
"""Extract stable public metadata while ignoring randomized page sections."""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.in_title = False
|
|
self.title_parts: list[str] = []
|
|
self.metadata: dict[str, str] = {}
|
|
|
|
def handle_starttag(
|
|
self,
|
|
tag: str,
|
|
attrs: list[tuple[str, str | None]],
|
|
) -> None:
|
|
attributes = {key.lower(): value or "" for key, value in attrs}
|
|
if tag.lower() == "title":
|
|
self.in_title = True
|
|
return
|
|
if tag.lower() == "meta":
|
|
key = (attributes.get("name") or attributes.get("property") or "").lower()
|
|
if key in {"description", "og:title", "og:description"}:
|
|
self.metadata[key] = attributes.get("content", "")[:4_000]
|
|
return
|
|
if tag.lower() == "link" and "canonical" in attributes.get("rel", "").lower():
|
|
self.metadata["canonical"] = attributes.get("href", "")[:2_000]
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag.lower() == "title":
|
|
self.in_title = False
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if self.in_title:
|
|
self.title_parts.append(data)
|
|
|
|
def stable_fields(self) -> dict[str, str]:
|
|
title = _clean_metadata_text(" ".join(self.title_parts))
|
|
fields = {
|
|
key: _clean_metadata_text(value)
|
|
for key, value in self.metadata.items()
|
|
if _clean_metadata_text(value)
|
|
}
|
|
if title:
|
|
fields["title"] = title
|
|
return fields
|
|
|
|
|
|
class LifecycleRepository(Protocol):
|
|
"""Persistence boundary used by production PostgreSQL and unit-test fakes."""
|
|
|
|
async def start_run(self, payload: dict[str, Any]) -> None:
|
|
...
|
|
|
|
async def latest_observations(self) -> dict[str, dict[str, Any]]:
|
|
...
|
|
|
|
async def save_observation(self, payload: dict[str, Any]) -> None:
|
|
...
|
|
|
|
async def finish_run(self, payload: dict[str, Any]) -> None:
|
|
...
|
|
|
|
|
|
class PostgresLifecycleRepository:
|
|
"""PostgreSQL repository with project-scoped RLS context."""
|
|
|
|
async def start_run(self, payload: dict[str, Any]) -> None:
|
|
async with get_db_context(PROJECT_ID) as db:
|
|
await db.execute(
|
|
text(
|
|
"""
|
|
INSERT INTO awooop_mcp_version_lifecycle_run (
|
|
run_id, trace_id, work_item_id, project_id, trigger_kind,
|
|
status, candidate_count, receipt, started_at
|
|
) VALUES (
|
|
:run_id, :trace_id, :work_item_id, :project_id,
|
|
:trigger_kind, 'running', :candidate_count,
|
|
CAST(:receipt AS jsonb), :started_at
|
|
)
|
|
"""
|
|
),
|
|
{
|
|
**payload,
|
|
"receipt": _json(payload["receipt"]),
|
|
},
|
|
)
|
|
|
|
async def latest_observations(self) -> dict[str, dict[str, Any]]:
|
|
async with get_db_context(PROJECT_ID) as db:
|
|
result = await db.execute(
|
|
text(
|
|
"""
|
|
SELECT DISTINCT ON (candidate_id)
|
|
candidate_id, observed_version, content_sha256,
|
|
discovery_status, observed_at
|
|
FROM awooop_mcp_version_observation
|
|
WHERE project_id = :project_id
|
|
AND discovery_status = 'observed'
|
|
ORDER BY candidate_id, observed_at DESC
|
|
"""
|
|
),
|
|
{"project_id": PROJECT_ID},
|
|
)
|
|
return {
|
|
str(row["candidate_id"]): dict(row) for row in result.mappings().all()
|
|
}
|
|
|
|
async def save_observation(self, payload: dict[str, Any]) -> None:
|
|
async with get_db_context(PROJECT_ID) as db:
|
|
await db.execute(
|
|
text(
|
|
"""
|
|
INSERT INTO awooop_mcp_version_observation (
|
|
run_id, trace_id, work_item_id, project_id, candidate_id,
|
|
catalog_source, source_url, source_url_sha256,
|
|
http_status, content_sha256, observed_version,
|
|
previous_version, discovery_status, change_state,
|
|
lifecycle_terminal, policy_decision, stage_receipts,
|
|
error_class, observed_at
|
|
) VALUES (
|
|
:run_id, :trace_id, :work_item_id, :project_id,
|
|
:candidate_id, :catalog_source, :source_url,
|
|
:source_url_sha256, :http_status, :content_sha256,
|
|
:observed_version, :previous_version, :discovery_status,
|
|
:change_state, :lifecycle_terminal,
|
|
CAST(:policy_decision AS jsonb),
|
|
CAST(:stage_receipts AS jsonb), :error_class,
|
|
:observed_at
|
|
)
|
|
"""
|
|
),
|
|
{
|
|
**payload,
|
|
"policy_decision": _json(payload["policy_decision"]),
|
|
"stage_receipts": _json(payload["stage_receipts"]),
|
|
},
|
|
)
|
|
|
|
async def finish_run(self, payload: dict[str, Any]) -> None:
|
|
async with get_db_context(PROJECT_ID) as db:
|
|
await db.execute(
|
|
text(
|
|
"""
|
|
UPDATE awooop_mcp_version_lifecycle_run
|
|
SET status = :status,
|
|
observed_count = :observed_count,
|
|
baseline_count = :baseline_count,
|
|
changed_count = :changed_count,
|
|
policy_blocked_count = :policy_blocked_count,
|
|
error_count = :error_count,
|
|
receipt = CAST(:receipt AS jsonb),
|
|
ended_at = :ended_at
|
|
WHERE run_id = :run_id AND project_id = :project_id
|
|
"""
|
|
),
|
|
{
|
|
**payload,
|
|
"receipt": _json(payload["receipt"]),
|
|
},
|
|
)
|
|
|
|
|
|
async def fetch_catalog_source(
|
|
source_url: str,
|
|
catalog_source: str,
|
|
timeout_seconds: int,
|
|
) -> CatalogFetchReceipt:
|
|
"""Fetch one allowlisted catalog URL without cross-domain redirects."""
|
|
allowed_hosts = _SOURCE_HOSTS.get(catalog_source)
|
|
if not allowed_hosts or not _is_allowed_url(source_url, allowed_hosts):
|
|
return CatalogFetchReceipt(
|
|
status="policy_blocked",
|
|
source_url=source_url,
|
|
error_class="catalog_url_not_allowlisted",
|
|
)
|
|
|
|
client = await get_general_client()
|
|
current_url = source_url
|
|
for redirect_count in range(_MAX_REDIRECTS + 1):
|
|
try:
|
|
async with client.stream(
|
|
"GET",
|
|
current_url,
|
|
follow_redirects=False,
|
|
timeout=httpx.Timeout(float(timeout_seconds), connect=5.0),
|
|
headers={
|
|
"Accept": "application/json,text/html,text/plain;q=0.9",
|
|
"User-Agent": "awoooi-mcp-version-lifecycle/1.0",
|
|
},
|
|
) as response:
|
|
if response.status_code in {301, 302, 303, 307, 308}:
|
|
location = response.headers.get("location")
|
|
if not location or redirect_count >= _MAX_REDIRECTS:
|
|
return CatalogFetchReceipt(
|
|
status="error",
|
|
source_url=current_url,
|
|
http_status=response.status_code,
|
|
error_class="redirect_limit_or_location_missing",
|
|
redirect_count=redirect_count,
|
|
)
|
|
redirected = urljoin(current_url, location)
|
|
if not _is_allowed_url(redirected, allowed_hosts):
|
|
return CatalogFetchReceipt(
|
|
status="policy_blocked",
|
|
source_url=current_url,
|
|
http_status=response.status_code,
|
|
error_class="cross_domain_redirect_blocked",
|
|
redirect_count=redirect_count,
|
|
)
|
|
current_url = redirected
|
|
continue
|
|
|
|
if response.status_code != 200:
|
|
return CatalogFetchReceipt(
|
|
status="error",
|
|
source_url=current_url,
|
|
http_status=response.status_code,
|
|
error_class=f"http_{response.status_code}",
|
|
redirect_count=redirect_count,
|
|
)
|
|
|
|
content_type = response.headers.get("content-type", "").lower()
|
|
if not any(
|
|
marker in content_type
|
|
for marker in (
|
|
"application/json",
|
|
"application/octet-stream",
|
|
"text/html",
|
|
"text/plain",
|
|
)
|
|
):
|
|
return CatalogFetchReceipt(
|
|
status="error",
|
|
source_url=current_url,
|
|
http_status=response.status_code,
|
|
error_class="unsupported_content_type",
|
|
redirect_count=redirect_count,
|
|
)
|
|
|
|
content_length = response.headers.get("content-length")
|
|
if content_length and int(content_length) > _MAX_SOURCE_BYTES:
|
|
return CatalogFetchReceipt(
|
|
status="error",
|
|
source_url=current_url,
|
|
http_status=response.status_code,
|
|
error_class="source_payload_too_large",
|
|
redirect_count=redirect_count,
|
|
)
|
|
|
|
body = bytearray()
|
|
async for chunk in response.aiter_bytes():
|
|
body.extend(chunk)
|
|
if len(body) > _MAX_SOURCE_BYTES:
|
|
return CatalogFetchReceipt(
|
|
status="error",
|
|
source_url=current_url,
|
|
http_status=response.status_code,
|
|
error_class="source_payload_too_large",
|
|
redirect_count=redirect_count,
|
|
)
|
|
return CatalogFetchReceipt(
|
|
status="ok",
|
|
source_url=current_url,
|
|
http_status=response.status_code,
|
|
body=bytes(body),
|
|
redirect_count=redirect_count,
|
|
)
|
|
except httpx.TimeoutException:
|
|
return CatalogFetchReceipt(
|
|
status="error",
|
|
source_url=current_url,
|
|
error_class="source_timeout",
|
|
redirect_count=redirect_count,
|
|
)
|
|
except (httpx.RequestError, ValueError):
|
|
return CatalogFetchReceipt(
|
|
status="error",
|
|
source_url=current_url,
|
|
error_class="source_network_error",
|
|
redirect_count=redirect_count,
|
|
)
|
|
|
|
return CatalogFetchReceipt(
|
|
status="error",
|
|
source_url=current_url,
|
|
error_class="redirect_loop",
|
|
redirect_count=_MAX_REDIRECTS,
|
|
)
|
|
|
|
|
|
async def run_mcp_version_lifecycle_once(
|
|
catalog: dict[str, Any],
|
|
*,
|
|
trigger_kind: str = "scheduled",
|
|
repository: LifecycleRepository | None = None,
|
|
fetcher: CatalogFetcher | None = None,
|
|
generated_at: datetime | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Observe every eligible external candidate and persist terminal receipts."""
|
|
repository = repository or PostgresLifecycleRepository()
|
|
fetcher = fetcher or fetch_catalog_source
|
|
now = generated_at or datetime.now(UTC)
|
|
run_id = uuid.uuid4()
|
|
trace_id = f"mcp-version-{run_id}"
|
|
work_item_id = f"MCP-VERSION-{now:%Y%m%dT%H%M%SZ}"
|
|
candidates = _eligible_candidates(catalog)
|
|
start_receipt = {
|
|
"schema_version": SCHEMA_VERSION,
|
|
"controller": "signal_worker_scheduled_fail_closed",
|
|
"raw_external_content_stored": False,
|
|
"external_install_performed": False,
|
|
"external_tool_call_performed": False,
|
|
"external_rag_write_performed": False,
|
|
"github_used": False,
|
|
"source_host_allowlist": sorted(
|
|
host for hosts in _SOURCE_HOSTS.values() for host in hosts
|
|
),
|
|
}
|
|
await repository.start_run(
|
|
{
|
|
"run_id": run_id,
|
|
"trace_id": trace_id,
|
|
"work_item_id": work_item_id,
|
|
"project_id": PROJECT_ID,
|
|
"trigger_kind": trigger_kind,
|
|
"candidate_count": len(candidates),
|
|
"receipt": start_receipt,
|
|
"started_at": now,
|
|
}
|
|
)
|
|
|
|
counts = {
|
|
"observed_count": 0,
|
|
"baseline_count": 0,
|
|
"changed_count": 0,
|
|
"policy_blocked_count": 0,
|
|
"error_count": 0,
|
|
}
|
|
terminals: dict[str, int] = {}
|
|
try:
|
|
previous = await repository.latest_observations()
|
|
|
|
semaphore = asyncio.Semaphore(_MAX_CONCURRENT_FETCHES)
|
|
|
|
async def fetch_candidate(
|
|
candidate: dict[str, Any],
|
|
) -> tuple[dict[str, Any], CatalogFetchReceipt]:
|
|
source_url = str(candidate["catalog_url"])
|
|
source_name = str(candidate["catalog_source"])
|
|
async with semaphore:
|
|
try:
|
|
fetched = await fetcher(
|
|
source_url,
|
|
source_name,
|
|
settings.MCP_VERSION_LIFECYCLE_SOURCE_TIMEOUT_SECONDS,
|
|
)
|
|
except Exception:
|
|
fetched = CatalogFetchReceipt(
|
|
status="error",
|
|
source_url=source_url,
|
|
error_class="source_fetcher_error",
|
|
)
|
|
return candidate, fetched
|
|
|
|
fetched_candidates = await asyncio.gather(
|
|
*(fetch_candidate(candidate) for candidate in candidates)
|
|
)
|
|
for candidate, fetched in fetched_candidates:
|
|
observation = build_candidate_observation(
|
|
candidate,
|
|
fetched=fetched,
|
|
previous=previous.get(str(candidate["capability_id"])),
|
|
run_id=run_id,
|
|
trace_id=trace_id,
|
|
work_item_id=work_item_id,
|
|
observed_at=now,
|
|
)
|
|
await repository.save_observation(observation)
|
|
_increment_counts(counts, observation)
|
|
terminal = str(observation["lifecycle_terminal"])
|
|
terminals[terminal] = terminals.get(terminal, 0) + 1
|
|
|
|
status = _run_status(counts)
|
|
ended_at = datetime.now(UTC)
|
|
final_receipt = {
|
|
**start_receipt,
|
|
"status": status,
|
|
"candidate_count": len(candidates),
|
|
**counts,
|
|
"terminal_counts": terminals,
|
|
"independent_post_verifier": {
|
|
"status": "verified",
|
|
"mutation_scope": "normalized_metadata_rows_only",
|
|
"raw_external_content_stored": False,
|
|
"external_install_performed": False,
|
|
"external_tool_call_performed": False,
|
|
"external_rag_write_performed": False,
|
|
"production_upgrade_performed": False,
|
|
},
|
|
"rollback": {
|
|
"status": "no_write_terminal",
|
|
"schema_rollback": "additive_tables_preserved",
|
|
},
|
|
}
|
|
await repository.finish_run(
|
|
{
|
|
"run_id": run_id,
|
|
"project_id": PROJECT_ID,
|
|
"status": status,
|
|
**counts,
|
|
"receipt": final_receipt,
|
|
"ended_at": ended_at,
|
|
}
|
|
)
|
|
logger.info(
|
|
"mcp_version_lifecycle_run_completed",
|
|
run_id=str(run_id),
|
|
status=status,
|
|
candidate_count=len(candidates),
|
|
**counts,
|
|
)
|
|
return {
|
|
"schema_version": SCHEMA_VERSION,
|
|
"run_id": str(run_id),
|
|
"trace_id": trace_id,
|
|
"work_item_id": work_item_id,
|
|
"status": status,
|
|
"candidate_count": len(candidates),
|
|
**counts,
|
|
"terminal_counts": terminals,
|
|
}
|
|
except Exception:
|
|
ended_at = datetime.now(UTC)
|
|
failure_receipt = {
|
|
**start_receipt,
|
|
"status": "failed",
|
|
"error_class": "controller_execution_error",
|
|
"independent_post_verifier": {
|
|
"status": "partial",
|
|
"external_install_performed": False,
|
|
"external_tool_call_performed": False,
|
|
"external_rag_write_performed": False,
|
|
"production_upgrade_performed": False,
|
|
},
|
|
"rollback": {"status": "no_write_terminal"},
|
|
}
|
|
try:
|
|
await repository.finish_run(
|
|
{
|
|
"run_id": run_id,
|
|
"project_id": PROJECT_ID,
|
|
"status": "failed",
|
|
**counts,
|
|
"receipt": failure_receipt,
|
|
"ended_at": ended_at,
|
|
}
|
|
)
|
|
except Exception:
|
|
logger.exception(
|
|
"mcp_version_lifecycle_failure_receipt_write_failed",
|
|
run_id=str(run_id),
|
|
)
|
|
raise
|
|
|
|
|
|
def build_candidate_observation(
|
|
candidate: dict[str, Any],
|
|
*,
|
|
fetched: CatalogFetchReceipt,
|
|
previous: dict[str, Any] | None,
|
|
run_id: uuid.UUID,
|
|
trace_id: str,
|
|
work_item_id: str,
|
|
observed_at: datetime,
|
|
) -> dict[str, Any]:
|
|
"""Normalize, diff, gate, and terminalize one source observation."""
|
|
candidate_id = str(candidate["capability_id"])
|
|
observed_version = _extract_unambiguous_version(fetched.body)
|
|
content_sha256, fingerprint_scope = _stable_catalog_fingerprint(
|
|
fetched.body,
|
|
source_url=fetched.source_url,
|
|
observed_version=observed_version,
|
|
)
|
|
previous_version = (
|
|
str(previous.get("observed_version"))
|
|
if previous and previous.get("observed_version")
|
|
else None
|
|
)
|
|
change_state = _change_state(
|
|
fetched=fetched,
|
|
content_sha256=content_sha256,
|
|
observed_version=observed_version,
|
|
previous=previous,
|
|
)
|
|
missing_receipts = _missing_supply_chain_receipts(candidate)
|
|
supply_chain_ready = not missing_receipts
|
|
|
|
if fetched.status != "ok":
|
|
discovery_status = (
|
|
"policy_blocked" if fetched.status == "policy_blocked" else "source_error"
|
|
)
|
|
terminal = "no_write_source_error"
|
|
replay_status = "not_started_source_unavailable"
|
|
elif change_state == "baseline_created":
|
|
discovery_status = "observed"
|
|
terminal = "no_write_baseline"
|
|
replay_status = "not_required_baseline_only"
|
|
elif change_state == "no_change":
|
|
discovery_status = "observed"
|
|
terminal = "no_write_no_change"
|
|
replay_status = "not_required_no_diff"
|
|
else:
|
|
discovery_status = "observed"
|
|
terminal = "no_write_policy_blocked"
|
|
replay_status = (
|
|
"blocked_registered_internal_replay_adapter_missing"
|
|
if supply_chain_ready
|
|
else "blocked_immutable_supply_chain_receipts_missing"
|
|
)
|
|
|
|
policy_decision = {
|
|
"risk_tier": str(candidate.get("risk_tier") or "high"),
|
|
"deployment_allowed_by_catalog": candidate.get("deployment_allowed") is True,
|
|
"immutable_supply_chain_ready": supply_chain_ready,
|
|
"missing_receipts": missing_receipts,
|
|
"registered_internal_replay_adapter": False,
|
|
"external_install_allowed": False,
|
|
"production_upgrade_allowed": False,
|
|
"rag_write_allowed": False,
|
|
}
|
|
stage_receipts = {
|
|
"sensor_source_receipt": {
|
|
"status": fetched.status,
|
|
"http_status": fetched.http_status,
|
|
"redirect_count": fetched.redirect_count,
|
|
"content_sha256": content_sha256,
|
|
"content_fingerprint_scope": fingerprint_scope,
|
|
"raw_body_stored": False,
|
|
},
|
|
"normalized_asset_identity": {
|
|
"candidate_id": candidate_id,
|
|
"catalog_source": str(candidate["catalog_source"]),
|
|
"source_url_sha256": hashlib.sha256(
|
|
fetched.source_url.encode("utf-8")
|
|
).hexdigest(),
|
|
},
|
|
"source_of_truth_diff": {
|
|
"change_state": change_state,
|
|
"previous_version": previous_version,
|
|
"observed_version": observed_version,
|
|
"previous_content_sha256": (
|
|
str(previous.get("content_sha256"))
|
|
if previous and previous.get("content_sha256")
|
|
else None
|
|
),
|
|
"observed_content_sha256": content_sha256,
|
|
},
|
|
"decision": {
|
|
"candidate_action": "observe_only_until_all_gates_pass",
|
|
"reason": replay_status,
|
|
},
|
|
"risk_policy": policy_decision,
|
|
"check_mode": {
|
|
"status": replay_status,
|
|
"external_process_started": False,
|
|
"artifact_downloaded": False,
|
|
},
|
|
"bounded_execution": {
|
|
"status": "not_executed",
|
|
"external_tool_call_performed": False,
|
|
"production_write_performed": False,
|
|
},
|
|
"post_verifier_and_rollback": {
|
|
"status": "verified_no_write_terminal",
|
|
"terminal": terminal,
|
|
"rollback_required": False,
|
|
"rollback_reason": "no_runtime_mutation_performed",
|
|
},
|
|
"learning_writeback": {
|
|
"status": "committed_with_observation_row",
|
|
"target": "awooop_mcp_version_observation",
|
|
"rag_or_km_write_performed": False,
|
|
},
|
|
}
|
|
return {
|
|
"run_id": run_id,
|
|
"trace_id": trace_id,
|
|
"work_item_id": work_item_id,
|
|
"project_id": PROJECT_ID,
|
|
"candidate_id": candidate_id,
|
|
"catalog_source": str(candidate["catalog_source"]),
|
|
"source_url": fetched.source_url,
|
|
"source_url_sha256": hashlib.sha256(
|
|
fetched.source_url.encode("utf-8")
|
|
).hexdigest(),
|
|
"http_status": fetched.http_status,
|
|
"content_sha256": content_sha256,
|
|
"observed_version": observed_version,
|
|
"previous_version": previous_version,
|
|
"discovery_status": discovery_status,
|
|
"change_state": change_state,
|
|
"lifecycle_terminal": terminal,
|
|
"policy_decision": policy_decision,
|
|
"stage_receipts": stage_receipts,
|
|
"error_class": fetched.error_class,
|
|
"observed_at": observed_at,
|
|
}
|
|
|
|
|
|
async def collect_mcp_version_lifecycle_readback() -> dict[str, Any]:
|
|
"""Return the newest durable run and bounded per-candidate terminal states."""
|
|
try:
|
|
async with get_db_context(PROJECT_ID) as db:
|
|
run_result = await db.execute(
|
|
text(
|
|
"""
|
|
SELECT
|
|
run_id, trace_id, work_item_id, trigger_kind, status,
|
|
candidate_count, observed_count, baseline_count,
|
|
changed_count, policy_blocked_count, error_count,
|
|
started_at, ended_at,
|
|
started_at >= NOW() - make_interval(
|
|
secs => :freshness_seconds
|
|
) AS fresh
|
|
FROM awooop_mcp_version_lifecycle_run
|
|
WHERE project_id = :project_id
|
|
ORDER BY started_at DESC
|
|
LIMIT 1
|
|
"""
|
|
),
|
|
{
|
|
"project_id": PROJECT_ID,
|
|
"freshness_seconds": (
|
|
settings.MCP_VERSION_LIFECYCLE_INTERVAL_SECONDS * 2
|
|
),
|
|
},
|
|
)
|
|
latest = run_result.mappings().one_or_none()
|
|
if latest is None:
|
|
return {
|
|
"status": "pending_first_run",
|
|
"controller_configured": True,
|
|
"controller_owner": "signal_worker",
|
|
"interval_seconds": settings.MCP_VERSION_LIFECYCLE_INTERVAL_SECONDS,
|
|
"latest_run": None,
|
|
"observations": [],
|
|
"blockers": ["first_runtime_receipt_pending"],
|
|
}
|
|
observation_result = await db.execute(
|
|
text(
|
|
"""
|
|
SELECT candidate_id, catalog_source, observed_version,
|
|
discovery_status, change_state, lifecycle_terminal,
|
|
error_class, observed_at
|
|
FROM awooop_mcp_version_observation
|
|
WHERE project_id = :project_id AND run_id = :run_id
|
|
ORDER BY candidate_id
|
|
LIMIT 64
|
|
"""
|
|
),
|
|
{"project_id": PROJECT_ID, "run_id": latest["run_id"]},
|
|
)
|
|
observations = [
|
|
{
|
|
"candidate_id": str(row["candidate_id"]),
|
|
"catalog_source": str(row["catalog_source"]),
|
|
"observed_version": row["observed_version"],
|
|
"discovery_status": str(row["discovery_status"]),
|
|
"change_state": str(row["change_state"]),
|
|
"lifecycle_terminal": str(row["lifecycle_terminal"]),
|
|
"error_class": row["error_class"],
|
|
"observed_at": _iso(row["observed_at"]),
|
|
}
|
|
for row in observation_result.mappings().all()
|
|
]
|
|
|
|
run_payload = {
|
|
"run_id": str(latest["run_id"]),
|
|
"trace_id": str(latest["trace_id"]),
|
|
"work_item_id": str(latest["work_item_id"]),
|
|
"trigger_kind": str(latest["trigger_kind"]),
|
|
"status": str(latest["status"]),
|
|
"candidate_count": int(latest["candidate_count"]),
|
|
"observed_count": int(latest["observed_count"]),
|
|
"baseline_count": int(latest["baseline_count"]),
|
|
"changed_count": int(latest["changed_count"]),
|
|
"policy_blocked_count": int(latest["policy_blocked_count"]),
|
|
"error_count": int(latest["error_count"]),
|
|
"started_at": _iso(latest["started_at"]),
|
|
"ended_at": _iso(latest["ended_at"]),
|
|
"fresh": bool(latest["fresh"]),
|
|
}
|
|
blockers = []
|
|
if not run_payload["fresh"]:
|
|
blockers.append("version_lifecycle_receipt_stale")
|
|
if run_payload["error_count"]:
|
|
blockers.append("catalog_source_observation_failures")
|
|
if run_payload["policy_blocked_count"]:
|
|
blockers.append("external_candidates_not_promotion_ready")
|
|
return {
|
|
"status": "ready" if run_payload["fresh"] else "degraded",
|
|
"controller_configured": True,
|
|
"controller_owner": "signal_worker",
|
|
"lock_backend": "redis_atomic_lease",
|
|
"interval_seconds": settings.MCP_VERSION_LIFECYCLE_INTERVAL_SECONDS,
|
|
"latest_run": run_payload,
|
|
"observations": observations,
|
|
"blockers": blockers,
|
|
}
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"mcp_version_lifecycle_readback_failed",
|
|
error_type=type(exc).__name__,
|
|
)
|
|
return {
|
|
"status": "degraded",
|
|
"controller_configured": True,
|
|
"controller_owner": "signal_worker",
|
|
"interval_seconds": settings.MCP_VERSION_LIFECYCLE_INTERVAL_SECONDS,
|
|
"latest_run": None,
|
|
"observations": [],
|
|
"blockers": ["version_lifecycle_schema_or_database_readback_failed"],
|
|
}
|
|
|
|
|
|
def _eligible_candidates(catalog: dict[str, Any]) -> list[dict[str, Any]]:
|
|
rows = catalog.get("capabilities")
|
|
if not isinstance(rows, list):
|
|
return []
|
|
candidates = [
|
|
row
|
|
for row in rows
|
|
if isinstance(row, dict)
|
|
and row.get("source_type") == "external"
|
|
and row.get("decision") != "rejected"
|
|
and row.get("catalog_source") in _SOURCE_HOSTS
|
|
and row.get("catalog_url")
|
|
]
|
|
return sorted(candidates, key=lambda row: str(row["capability_id"]))
|
|
|
|
|
|
def _is_allowed_url(url: str, allowed_hosts: frozenset[str]) -> bool:
|
|
parsed = urlparse(url)
|
|
try:
|
|
port = parsed.port
|
|
except ValueError:
|
|
return False
|
|
return (
|
|
parsed.scheme == "https"
|
|
and (parsed.hostname or "").lower() in allowed_hosts
|
|
and parsed.username is None
|
|
and parsed.password is None
|
|
and port in {None, 443}
|
|
)
|
|
|
|
|
|
def _extract_unambiguous_version(body: bytes) -> str | None:
|
|
versions = {
|
|
match.group(1).decode("ascii")
|
|
for pattern in _VERSION_PATTERNS
|
|
for match in pattern.finditer(body)
|
|
}
|
|
return next(iter(versions)) if len(versions) == 1 else None
|
|
|
|
|
|
def _stable_catalog_fingerprint(
|
|
body: bytes,
|
|
*,
|
|
source_url: str,
|
|
observed_version: str | None,
|
|
) -> tuple[str | None, str]:
|
|
"""Hash stable identity metadata instead of randomized recommendation HTML."""
|
|
if not body:
|
|
return None, "not_available"
|
|
|
|
parser = _StableMetadataParser()
|
|
try:
|
|
parser.feed(body.decode("utf-8", errors="ignore"))
|
|
except (UnicodeError, ValueError):
|
|
stable_fields: dict[str, str] = {}
|
|
else:
|
|
stable_fields = parser.stable_fields()
|
|
if stable_fields:
|
|
normalized = {
|
|
"source_url": source_url,
|
|
"observed_version": observed_version,
|
|
"metadata": stable_fields,
|
|
}
|
|
return hashlib.sha256(_json(normalized).encode("utf-8")).hexdigest(), (
|
|
"title_description_canonical_version"
|
|
)
|
|
return hashlib.sha256(body).hexdigest(), "bounded_raw_fallback"
|
|
|
|
|
|
def _clean_metadata_text(value: str) -> str:
|
|
return " ".join(html.unescape(value).split())[:4_000]
|
|
|
|
|
|
def _change_state(
|
|
*,
|
|
fetched: CatalogFetchReceipt,
|
|
content_sha256: str | None,
|
|
observed_version: str | None,
|
|
previous: dict[str, Any] | None,
|
|
) -> str:
|
|
if fetched.status != "ok":
|
|
return "source_unavailable"
|
|
if previous is None:
|
|
return "baseline_created"
|
|
previous_version = previous.get("observed_version")
|
|
if previous_version and observed_version and previous_version != observed_version:
|
|
return "version_changed"
|
|
if previous.get("content_sha256") != content_sha256:
|
|
return "content_changed"
|
|
return "no_change"
|
|
|
|
|
|
def _missing_supply_chain_receipts(candidate: dict[str, Any]) -> list[str]:
|
|
receipts = candidate.get("supply_chain_receipts")
|
|
receipts = receipts if isinstance(receipts, dict) else {}
|
|
required = (
|
|
"exact_version",
|
|
"immutable_artifact_digest",
|
|
"internal_mirror_ref",
|
|
"sbom_ref",
|
|
"signature_or_provenance_ref",
|
|
"license_decision",
|
|
"vulnerability_decision",
|
|
"rollback_artifact_ref",
|
|
)
|
|
missing = [key for key in required if not receipts.get(key)]
|
|
if candidate.get("deployment_allowed") is not True:
|
|
missing.append("deployment_policy_not_enabled")
|
|
return missing
|
|
|
|
|
|
def _increment_counts(counts: dict[str, int], observation: dict[str, Any]) -> None:
|
|
if observation["discovery_status"] == "observed":
|
|
counts["observed_count"] += 1
|
|
else:
|
|
counts["error_count"] += 1
|
|
if observation["change_state"] == "baseline_created":
|
|
counts["baseline_count"] += 1
|
|
if observation["change_state"] in {"version_changed", "content_changed"}:
|
|
counts["changed_count"] += 1
|
|
if observation["policy_decision"]["immutable_supply_chain_ready"] is False:
|
|
counts["policy_blocked_count"] += 1
|
|
|
|
|
|
def _run_status(counts: dict[str, int]) -> str:
|
|
if counts["error_count"]:
|
|
return "partial_degraded"
|
|
if counts["policy_blocked_count"]:
|
|
return "completed_no_write_policy_blocked"
|
|
if counts["changed_count"] == 0:
|
|
return "completed_no_change"
|
|
return "completed_no_write_policy_blocked"
|
|
|
|
|
|
def _json(payload: dict[str, Any]) -> str:
|
|
return json.dumps(
|
|
payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
|
)
|
|
|
|
|
|
def _iso(value: Any) -> str | None:
|
|
return value.isoformat() if hasattr(value, "isoformat") else None
|