wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(aiops): gate StockPlatform cron repair closure

Browse Source
This commit is contained in:
Your Name
2026-07-22 18:18:30 +08:00
parent 979d52c81f
commit d4fd87db31
8 changed files with 1408 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
apps/api/tests/test_independent_verifier_registry.py
View File

@@ -71,6 +71,21 @@ def test_registry_covers_every_typed_domain_without_false_runtime_closure() -> N
assert backup_gate["required_source"] == (
"same_run_backup_freshness_escrow_restore_metadata"
)
stock_cron = {
row["readback_id"]: row
for row in registry["external_readbacks"]
if row["readback_id"].startswith("stockplatform_cron_")
}
assert set(stock_cron) == {
"stockplatform_cron_exit_code_postcondition",
"stockplatform_cron_freshness_postcondition",
}
assert stock_cron["stockplatform_cron_exit_code_postcondition"][
"required_source"
] == "stockplatform_cron_scheduler_readback"
assert stock_cron["stockplatform_cron_freshness_postcondition"][
"required_source"
] == "stockplatform_public_api_freshness_readback"
def test_backup_readback_route_matches_independent_verifier_contract() -> None:

10
apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py
View File

@@ -155,9 +155,9 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
"active_or_completed_commitments": 68,
"by_status": {
"analysis_or_governance_complete": 6,
"source_implemented_runtime_pending": 33,
"source_implemented_runtime_pending": 34,
"in_progress": 27,
"not_started_or_no_current_evidence": 2,
"not_started_or_no_current_evidence": 1,
"superseded": 2,
},
"product_runtime_closed_commitments": 0,
@@ -200,6 +200,12 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
assert "DR scorecard remains blocked" in " ".join(
commitments["AIA-CONV-035"]["source_evidence"]
)
assert commitments["AIA-CONV-041"]["status"] == (
"source_implemented_runtime_pending"
)
assert "independent exit-code and freshness receipts" in " ".join(
commitments["AIA-CONV-041"]["source_evidence"]
)
assert "Host112" in commitments["AIA-CONV-049"]["title"]
assert commitments["AIA-CONV-049"]["status"] == (
"source_implemented_runtime_pending"

396
apps/api/tests/test_stockplatform_cron_intelligence_sync_control.py Normal file
View File

@@ -0,0 +1,396 @@
from __future__ import annotations
# ruff: noqa: E402
import os
from pathlib import Path
import pytest
os.environ.setdefault(
"DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test"
)
from src.services.service_registry import ServiceRegistryClient # noqa: E402
from src.services.stockplatform_cron_intelligence_sync_control import ( # noqa: E402
CANONICAL_ASSET_ID,
EXIT_CODE_VERIFIER,
FRESHNESS_VERIFIER,
build_stockplatform_cron_closure,
build_stockplatform_cron_repair_candidate,
record_stockplatform_cron_closure,
)
ROOT = Path(__file__).resolve().parents[3]
REGISTRY_PATH = ROOT / "ops" / "config" / "service-registry.yaml"
SOURCE_RUN_ID = "6a81ca92-8cc7-4af7-b84c-d988ecb4552b"
EXECUTION_RUN_ID = "786defde-85fc-49ca-97b6-6456ca8ed69d"
TRACE_ID = "stockplatform-cron-trace-041"
WORK_ITEM_ID = "AIA-STOCK-CRON-041"
LOCAL_UNPUSHED_SHA = "af9a47379a2e26198b68fff0b8b423417b3e9a70"
class _CandidateStore:
def __init__(self) -> None:
self.records: dict[str, dict] = {}
async def __call__(self, record: dict, project_id: str) -> dict:
assert project_id == "awoooi"
fingerprint = record["fingerprint"]
existing = self.records.get(fingerprint)
if existing is not None:
return {
"event_id": f"candidate:{fingerprint[:12]}",
"created": False,
"record": existing,
}
self.records[fingerprint] = record
return {
"event_id": f"candidate:{fingerprint[:12]}",
"created": True,
"record": record,
}
class _ClosureStore:
def __init__(self) -> None:
self.records: dict[str, dict] = {}
async def __call__(self, record: dict, project_id: str) -> dict:
assert project_id == "awoooi"
receipt_id = record["receipt_id"]
existing = self.records.get(receipt_id)
if existing is not None:
return {
"event_id": f"closure:{receipt_id}",
"created": False,
"record": existing,
}
self.records[receipt_id] = record
return {
"event_id": f"closure:{receipt_id}",
"created": True,
"record": record,
}
def _base(receipt_id: str) -> dict:
return {
"receipt_id": receipt_id,
"trace_id": TRACE_ID,
"run_id": SOURCE_RUN_ID,
"work_item_id": WORK_ITEM_ID,
"durable_readback_ack": True,
"freshness_verified": True,
"max_age_seconds": 300,
"observed_at": "2026-07-22T05:00:00+00:00",
}
def _failure() -> dict:
return {
**_base("stock-cron-failure:041"),
"schema_version": "stockplatform_cron_failure_receipt_v1",
"source": "stockplatform_cron_runtime_observer",
"product_id": "stockplatform-v2",
"job_name": "cron_intelligence_sync",
"canonical_asset_id": CANONICAL_ASSET_ID,
"consecutive_failures": 4,
"last_exit_code": 1,
}
def _diagnosis() -> dict:
return {
**_base("stock-cron-rca:041"),
"schema_version": "stockplatform_cron_wrapped_command_rca_receipt_v1",
"source": "sanitized_stockplatform_cron_rca",
"product_id": "stockplatform-v2",
"job_name": "cron_intelligence_sync",
"canonical_asset_id": CANONICAL_ASSET_ID,
"root_cause": "wrapped_command_exit_code_not_propagated",
"root_cause_verified": True,
"sanitized": True,
"untrusted_evidence": True,
"raw_command_recorded": False,
"raw_log_recorded": False,
"evidence_refs": ["cron-run:041", "wrapped-command:041"],
}
def _source_fix(*, pushed: bool = False, reachable: bool = False) -> dict:
return {
**_base("stock-cron-source-fix:041"),
"schema_version": "stockplatform_cron_source_fix_receipt_v1",
"source": "stockplatform_source_change_receipt",
"repository": "wooo/stockplatform-v2",
"product_id": "stockplatform-v2",
"job_name": "cron_intelligence_sync",
"canonical_asset_id": CANONICAL_ASSET_ID,
"commit_sha": LOCAL_UNPUSHED_SHA,
"fix_kind": "wrapped_command_exit_code_propagation",
"wrapped_command_fix_present": True,
"focused_tests_passed": True,
"raw_patch_recorded": False,
"pushed_to_gitea": pushed,
"commit_reachable_from_gitea_main": reachable,
}
async def _candidate(store: _CandidateStore) -> tuple[dict, dict]:
result = await build_stockplatform_cron_repair_candidate(
target_resource="cron_intelligence_sync",
failure_receipt=_failure(),
diagnosis_receipt=_diagnosis(),
source_fix_receipt=_source_fix(),
registry=ServiceRegistryClient(REGISTRY_PATH),
persist_candidate=store,
)
return result, store.records[result["fingerprint"]]
def _deployment(candidate: dict) -> dict:
return {
"schema_version": "stockplatform_cron_deployment_receipt_v1",
"receipt_id": "stock-cron-deploy:041",
"source": "gitea_controlled_cd_release_readback",
"executor": "gitea_controlled_cd_executor",
"status": "deployed",
"durable_readback_ack": True,
"production_deploy": True,
"repository": "wooo/stockplatform-v2",
"commit_sha": LOCAL_UNPUSHED_SHA,
"canonical_asset_id": CANONICAL_ASSET_ID,
"job_name": "cron_intelligence_sync",
"candidate_fingerprint": candidate["fingerprint"],
"execution_run_id": EXECUTION_RUN_ID,
"deployed_at": "2026-07-22T06:00:00+00:00",
}
def _exit_code(candidate: dict, deployment: dict) -> dict:
return {
"schema_version": "stockplatform_cron_exit_code_verifier_receipt_v1",
"receipt_id": "stock-cron-exit:041",
"verifier": EXIT_CODE_VERIFIER,
"source": "stockplatform_cron_scheduler_readback",
"status": "verified",
"durable_readback_ack": True,
"freshness_verified": True,
"max_age_seconds": 300,
"execution_run_id": EXECUTION_RUN_ID,
"candidate_fingerprint": candidate["fingerprint"],
"deployment_receipt_id": deployment["receipt_id"],
"canonical_asset_id": CANONICAL_ASSET_ID,
"job_name": "cron_intelligence_sync",
"process_exit_code": 0,
"wrapped_command_exit_code_propagated": True,
"invocation_count": 1,
"active_blockers": [],
"runtime_mutation_performed": False,
"observed_at": "2026-07-22T06:05:00+00:00",
}
def _freshness(candidate: dict, deployment: dict) -> dict:
return {
"schema_version": "stockplatform_cron_freshness_verifier_receipt_v1",
"receipt_id": "stock-cron-freshness:041",
"verifier": FRESHNESS_VERIFIER,
"source": "stockplatform_public_api_freshness_readback",
"status": "verified",
"durable_readback_ack": True,
"freshness_verified": True,
"max_age_seconds": 300,
"execution_run_id": EXECUTION_RUN_ID,
"candidate_fingerprint": candidate["fingerprint"],
"deployment_receipt_id": deployment["receipt_id"],
"canonical_asset_id": CANONICAL_ASSET_ID,
"job_name": "cron_intelligence_sync",
"latest_success_at": "2026-07-22T06:07:00+00:00",
"observed_at": "2026-07-22T06:08:00+00:00",
"age_seconds": 60,
"freshness_slo_seconds": 900,
"active_blockers": [],
"runtime_mutation_performed": False,
}
def test_registry_resolves_only_the_exact_stockplatform_schedule_identity() -> None:
registry = ServiceRegistryClient(REGISTRY_PATH)
exact = registry.resolve_identity("cron_intelligence_sync")
unknown = registry.resolve_identity("cron_intelligence_sync_guess")
assert exact["canonical_id"] == CANONICAL_ASSET_ID
assert exact["asset_domain"] == "docker_container"
assert exact["host"] == "192.168.0.110"
assert exact["executor"] == "host_ansible_executor"
assert exact["verifier"] == "stockplatform_cron_independent_verifier"
assert exact["controlled_apply_allowed"] is False
assert unknown["resolution_status"] == "asset_identity_unresolved"
assert unknown["controlled_apply_allowed"] is False
@pytest.mark.asyncio
async def test_local_unpushed_fix_creates_one_durable_release_pending_candidate() -> None:
store = _CandidateStore()
first, record = await _candidate(store)
second, _ = await _candidate(store)
assert first["status"] == "source_fix_candidate_recorded_release_pending"
assert first["created"] is True
assert first["candidate_created"] is True
assert first["controlled_release_allowed"] is False
assert first["active_blockers"] == [
"source_fix_not_reachable_from_gitea_main"
]
assert first["executor_invoked"] is False
assert first["agent99_dispatch_performed"] is False
assert first["provider_call_performed"] is False
assert first["runtime_mutation_performed"] is False
assert second["deduplicated"] is True
assert second["fingerprint"] == first["fingerprint"]
assert len(store.records) == 1
assert record["source_fix"] == {
"repository": "wooo/stockplatform-v2",
"commit_sha": LOCAL_UNPUSHED_SHA,
"fix_kind": "wrapped_command_exit_code_propagation",
"focused_tests_passed": True,
"pushed_to_gitea": False,
"commit_reachable_from_gitea_main": False,
"runtime_verified": False,
}
assert record["policy"]["direct_executor_invocation_allowed"] is False
@pytest.mark.asyncio
async def test_unknown_schedule_creates_only_durable_identity_drift_item() -> None:
store = _CandidateStore()
result = await build_stockplatform_cron_repair_candidate(
target_resource="unknown-stock-cron",
failure_receipt=_failure(),
diagnosis_receipt=_diagnosis(),
source_fix_receipt=_source_fix(),
registry=ServiceRegistryClient(REGISTRY_PATH),
persist_candidate=store,
)
assert result["status"] == "asset_identity_unresolved"
assert result["candidate_created"] is False
assert result["controlled_release_allowed"] is False
assert result["typed_domain"] == "unknown"
assert result["work_item_id"].startswith("AIA-ASSET-DRIFT-")
assert result["active_blockers"] == ["canonical_asset_identity_unresolved"]
assert result["executor_invoked"] is False
assert result["cross_domain_fallback_allowed"] is False
assert len(store.records) == 1
record = next(iter(store.records.values()))
assert record["kind"] == "asset_identity_drift_work_item"
assert record["source_fix"] == {}
@pytest.mark.asyncio
async def test_unverified_or_raw_wrapped_command_evidence_never_persists_candidate() -> None:
store = _CandidateStore()
diagnosis = _diagnosis()
diagnosis["raw_command_recorded"] = True
diagnosis["root_cause_verified"] = False
result = await build_stockplatform_cron_repair_candidate(
target_resource="cron_intelligence_sync",
failure_receipt=_failure(),
diagnosis_receipt=diagnosis,
source_fix_receipt=_source_fix(),
registry=ServiceRegistryClient(REGISTRY_PATH),
persist_candidate=store,
)
assert result["status"] == "source_fix_candidate_blocked"
assert result["active_blockers"] == [
"wrapped_command_root_cause_unverified"
]
assert result["candidate_created"] is False
assert result["runtime_mutation_performed"] is False
assert store.records == {}
@pytest.mark.asyncio
async def test_closure_fails_closed_on_missing_mismatched_or_stale_verifier() -> None:
_, candidate = await _candidate(_CandidateStore())
deployment = _deployment(candidate)
exit_code = _exit_code(candidate, deployment)
freshness = _freshness(candidate, deployment)
mismatched_exit = {**exit_code, "execution_run_id": SOURCE_RUN_ID}
result = build_stockplatform_cron_closure(
candidate=candidate,
deployment_receipt=deployment,
exit_code_receipt=mismatched_exit,
freshness_receipt=freshness,
)
assert result["runtime_closed"] is False
assert "exit_code_independent_verifier_not_closed" in result["active_blockers"]
stale_freshness = {
**freshness,
"latest_success_at": "2026-07-22T05:59:00+00:00",
}
result = build_stockplatform_cron_closure(
candidate=candidate,
deployment_receipt=deployment,
exit_code_receipt=exit_code,
freshness_receipt=stale_freshness,
)
assert result["runtime_closed"] is False
assert "freshness_independent_verifier_not_closed" in result["active_blockers"]
wrong_commit = {**deployment, "commit_sha": "0" * 40}
result = build_stockplatform_cron_closure(
candidate=candidate,
deployment_receipt=wrong_commit,
exit_code_receipt=exit_code,
freshness_receipt=freshness,
)
assert result["runtime_closed"] is False
assert "exact_controlled_deployment_receipt_unverified" in result[
"active_blockers"
]
assert result["durable_closure_written"] is False
@pytest.mark.asyncio
async def test_same_run_dual_verifiers_create_one_durable_closure_receipt() -> None:
_, candidate = await _candidate(_CandidateStore())
deployment = _deployment(candidate)
exit_code = _exit_code(candidate, deployment)
freshness = _freshness(candidate, deployment)
store = _ClosureStore()
first = await record_stockplatform_cron_closure(
candidate=candidate,
deployment_receipt=deployment,
exit_code_receipt=exit_code,
freshness_receipt=freshness,
persist_closure=store,
)
second = await record_stockplatform_cron_closure(
candidate=candidate,
deployment_receipt=deployment,
exit_code_receipt=exit_code,
freshness_receipt=freshness,
persist_closure=store,
)
assert first["status"] == "runtime_closed"
assert first["runtime_closed"] is True
assert first["durable_closure_written"] is True
assert first["created"] is True
assert first["active_blockers"] == []
assert first["independent_verifiers"] == [
EXIT_CODE_VERIFIER,
FRESHNESS_VERIFIER,
]
assert first["executor"] not in first["independent_verifiers"]
assert first["runtime_mutation_performed_by_verifier"] is False
assert second["deduplicated"] is True
assert second["receipt_id"] == first["receipt_id"]
assert len(store.records) == 1