fix(automation): preserve immutable learning receipts

2026-07-15 11:19:56 +08:00
parent 9476b234fc
commit 61ff8eff4e
2 changed files with 233 additions and 21 deletions
--- a/apps/api/src/services/awooop_ansible_learning_writeback.py
+++ b/apps/api/src/services/awooop_ansible_learning_writeback.py
@@ -20,6 +20,7 @@ from src.utils.timezone import now_taipei
 logger = structlog.get_logger(__name__)

 _CANONICAL_PLAYBOOK_PREFIX = "PB-ANSIBLE-"
+_SHA256_HEX_RE = re.compile(r"^[0-9a-f]{64}$")


 def canonical_ansible_playbook_id(catalog_id: str) -> str:
@@ -174,48 +175,88 @@ def _playbook_trust_readback(


 def _existing_trust_readback_matches(
+    existing_input: dict[str, Any],
    existing_trust: dict[str, Any],
    *,
    playbook: PlaybookRecord,
    identity: dict[str, Any],
+    automation_run_id: str,
+    incident_id: str,
+    catalog_id: str,
+    apply_op_id: str,
+    verification_result: str,
    trust_mutation_performed: bool = True,
    observation_kind: str = "controlled_apply",
 ) -> bool:
+    """Verify an immutable receipt without rebinding it to mutable state.
+
+    A receipt fingerprints the PlayBook row observed by its own operation. The
+    canonical PlayBook can legitimately evolve later, so comparing a historical
+    fingerprint with the current row makes a valid receipt fail forever and
+    starves the bounded backfill queue.
+    """
+
    expected_schema_version = (
        "ansible_playbook_trust_writeback_v2"
        if trust_mutation_performed
        else "ansible_playbook_no_write_observation_v1"
    )
+    receipt_version = existing_trust.get("playbook_row_version")
+    try:
+        receipt_version = int(receipt_version)
+    except (TypeError, ValueError):
+        return False
+    current_version = int(playbook.version or 0)
+    receipt_fingerprint = str(
+        existing_trust.get("playbook_row_fingerprint") or ""
+    )
    return bool(
-        existing_trust.get("schema_version")
+        existing_input.get("schema_version")
+        == "ansible_learning_writeback_receipt_v3"
+        and existing_input.get("automation_run_id") == automation_run_id
+        and existing_input.get("incident_id") == incident_id
+        and existing_input.get("catalog_id") == catalog_id
+        and existing_input.get("canonical_playbook_id")
+        == identity["canonical_playbook_id"]
+        and existing_input.get("identity_schema_version")
+        == identity["schema_version"]
+        and existing_input.get("identity_fingerprint")
+        == identity["identity_fingerprint"]
+        and existing_input.get("playbook_path") == identity["playbook_path"]
+        and existing_input.get("apply_op_id") == apply_op_id
+        and existing_input.get("verification_result") == verification_result
+        and existing_input.get("observation_kind") == observation_kind
+        and existing_input.get("trust_mutation_performed")
+        is trust_mutation_performed
+        and existing_input.get("learning_repository") == "playbooks"
+        and existing_input.get("stores_raw_logs") is False
+        and existing_input.get("stores_secret_values") is False
+        and existing_trust.get("schema_version")
        == expected_schema_version
        and existing_trust.get("canonical_playbook_id")
        == identity["canonical_playbook_id"]
+        and existing_trust.get("catalog_id") == catalog_id
+        and existing_trust.get("playbook_path") == identity["playbook_path"]
+        and existing_trust.get("identity_schema_version")
+        == identity["schema_version"]
        and existing_trust.get("identity_fingerprint")
        == identity["identity_fingerprint"]
-        and existing_trust.get("playbook_row_version")
-        == int(playbook.version or 0)
-        and existing_trust.get("playbook_row_fingerprint")
-        == _playbook_row_fingerprint(playbook)
+        and receipt_version > 0
+        and current_version >= receipt_version
+        and _SHA256_HEX_RE.fullmatch(receipt_fingerprint) is not None
+        and existing_trust.get("verification_result") == verification_result
        and existing_trust.get("repository_readback_verified") is True
+        and existing_trust.get("repository_write_acknowledged") is True
        and existing_trust.get("operation_receipt_readback_verified") is True
+        and existing_trust.get("durable_write_acknowledged") is True
        and existing_trust.get("learning_recorded") is True
+        and existing_trust.get("trust_observation_recorded") is True
        and existing_trust.get("trust_updated") is trust_mutation_performed
-        and (
-            existing_trust.get("trust_mutation_performed")
-            is trust_mutation_performed
-            or (
-                trust_mutation_performed
-                and existing_trust.get("trust_mutation_performed") is None
-            )
-        )
-        and (
-            existing_trust.get("observation_kind") == observation_kind
-            or (
-                observation_kind == "controlled_apply"
-                and existing_trust.get("observation_kind") is None
-            )
-        )
+        and existing_trust.get("trust_mutation_performed")
+        is trust_mutation_performed
+        and existing_trust.get("observation_kind") == observation_kind
+        and existing_trust.get("raw_log_payload_stored") is False
+        and existing_trust.get("secret_value_stored") is False
    )


@@ -270,6 +311,9 @@ async def record_ansible_playbook_trust_writeback(
            existing_output = _json_dict(
                existing_row.get("output") if existing_row else None
            )
+            existing_input = _json_dict(
+                existing_row.get("input") if existing_row else None
+            )
            existing_trust = _json_dict(
                existing_output.get("playbook_trust_writeback")
            )
@@ -290,9 +334,15 @@ async def record_ansible_playbook_trust_writeback(
                    )
                    return None
                if _existing_trust_readback_matches(
+                    existing_input,
                    existing_trust,
                    playbook=playbook,
                    identity=identity,
+                    automation_run_id=automation_run_id,
+                    incident_id=incident_id,
+                    catalog_id=catalog_id,
+                    apply_op_id=apply_op_id,
+                    verification_result=verification_result,
                    trust_mutation_performed=trust_mutation_performed,
                    observation_kind=observation_kind,
                ):
@@ -307,7 +357,8 @@ async def record_ansible_playbook_trust_writeback(
                    ),
                )
                return None
-            if playbook is None:
+            playbook_created = playbook is None
+            if playbook_created:
                playbook = PlaybookRecord(
                    playbook_id=canonical_id,
                    project_id=project_id,
@@ -372,6 +423,11 @@ async def record_ansible_playbook_trust_writeback(
                await db.flush()

            if trust_mutation_performed:
+                if not playbook_created:
+                    playbook.version = int(playbook.version or 0) + 1
+                    playbook.version_reason = (
+                        "ansible_trust_observation:" + apply_op_id
+                    )
                _record_catalog_revision_observation(playbook, catalog)
                source_incident_ids = list(playbook.source_incident_ids or [])
                if incident_id and incident_id not in source_incident_ids:
--- a/apps/api/tests/test_awooop_ansible_learning_writeback.py
+++ b/apps/api/tests/test_awooop_ansible_learning_writeback.py
@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from types import SimpleNamespace
+
+import pytest
+
+from src.services.awooop_ansible_learning_writeback import (
+    _existing_trust_readback_matches,
+)
+
+
+def _learning_receipt_fixture() -> (
+    tuple[
+        SimpleNamespace,
+        dict[str, object],
+        dict[str, object],
+        dict[str, object],
+    ]
+):
+    playbook = SimpleNamespace(
+        playbook_id="PB-ANSIBLE-110-DISK-PRESSURE",
+        project_id="awoooi",
+        version=1,
+        success_count=19,
+        failure_count=0,
+        trust_score=0.9,
+        symptom_pattern={"catalog_revision": "disk-pressure-v1"},
+        updated_at=datetime(2026, 7, 15, tzinfo=UTC),
+    )
+    identity: dict[str, object] = {
+        "schema_version": "ansible_playbook_identity_v1",
+        "identity_fingerprint": "a" * 64,
+        "canonical_playbook_id": playbook.playbook_id,
+        "catalog_id": "ansible:110-disk-pressure",
+        "playbook_path": (
+            "infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
+        ),
+    }
+    receipt_input: dict[str, object] = {
+        "schema_version": "ansible_learning_writeback_receipt_v3",
+        "automation_run_id": "run-1",
+        "incident_id": "INC-1",
+        "catalog_id": identity["catalog_id"],
+        "canonical_playbook_id": identity["canonical_playbook_id"],
+        "identity_schema_version": identity["schema_version"],
+        "identity_fingerprint": identity["identity_fingerprint"],
+        "playbook_path": identity["playbook_path"],
+        "apply_op_id": "11111111-1111-4111-8111-111111111111",
+        "verification_result": "success",
+        "observation_kind": "controlled_apply",
+        "trust_mutation_performed": True,
+        "learning_repository": "playbooks",
+        "stores_raw_logs": False,
+        "stores_secret_values": False,
+    }
+    trust_receipt: dict[str, object] = {
+        "schema_version": "ansible_playbook_trust_writeback_v2",
+        "identity_schema_version": identity["schema_version"],
+        "identity_fingerprint": identity["identity_fingerprint"],
+        "canonical_playbook_id": identity["canonical_playbook_id"],
+        "catalog_id": identity["catalog_id"],
+        "playbook_path": identity["playbook_path"],
+        "playbook_row_version": 1,
+        "playbook_row_fingerprint": "b" * 64,
+        "verification_result": "success",
+        "observation_kind": "controlled_apply",
+        "learning_recorded": True,
+        "trust_observation_recorded": True,
+        "trust_updated": True,
+        "trust_mutation_performed": True,
+        "repository_write_acknowledged": True,
+        "repository_readback_verified": True,
+        "operation_receipt_readback_verified": True,
+        "durable_write_acknowledged": True,
+        "raw_log_payload_stored": False,
+        "secret_value_stored": False,
+    }
+    return playbook, identity, receipt_input, trust_receipt
+
+
+def _matches(
+    playbook: SimpleNamespace,
+    identity: dict[str, object],
+    receipt_input: dict[str, object],
+    trust_receipt: dict[str, object],
+) -> bool:
+    return _existing_trust_readback_matches(
+        receipt_input,
+        trust_receipt,
+        playbook=playbook,
+        identity=identity,
+        automation_run_id="run-1",
+        incident_id="INC-1",
+        catalog_id="ansible:110-disk-pressure",
+        apply_op_id="11111111-1111-4111-8111-111111111111",
+        verification_result="success",
+    )
+
+
+def test_historical_receipt_survives_later_playbook_mutation() -> None:
+    playbook, identity, receipt_input, trust_receipt = _learning_receipt_fixture()
+
+    assert _matches(playbook, identity, receipt_input, trust_receipt)
+
+    playbook.version = 3
+    playbook.success_count = 21
+    playbook.updated_at = datetime(2026, 7, 16, tzinfo=UTC)
+
+    assert _matches(playbook, identity, receipt_input, trust_receipt)
+
+
+def test_historical_receipt_accepts_legacy_same_version_row_drift() -> None:
+    playbook, identity, receipt_input, trust_receipt = _learning_receipt_fixture()
+    playbook.success_count = 20
+    playbook.updated_at = datetime(2026, 7, 16, tzinfo=UTC)
+
+    assert _matches(playbook, identity, receipt_input, trust_receipt)
+
+
+@pytest.mark.parametrize(
+    ("field", "wrong_value"),
+    [
+        ("automation_run_id", "wrong-run"),
+        ("incident_id", "INC-WRONG"),
+        ("apply_op_id", "22222222-2222-4222-8222-222222222222"),
+        ("verification_result", "failed"),
+    ],
+)
+def test_historical_receipt_rejects_wrong_operation_binding(
+    field: str,
+    wrong_value: str,
+) -> None:
+    playbook, identity, receipt_input, trust_receipt = _learning_receipt_fixture()
+    receipt_input[field] = wrong_value
+
+    assert not _matches(playbook, identity, receipt_input, trust_receipt)
+
+
+@pytest.mark.parametrize(
+    ("field", "wrong_value"),
+    [
+        ("playbook_row_version", 0),
+        ("playbook_row_fingerprint", "not-a-sha256"),
+        ("repository_readback_verified", False),
+        ("operation_receipt_readback_verified", False),
+    ],
+)
+def test_historical_receipt_rejects_weak_durable_readback(
+    field: str,
+    wrong_value: object,
+) -> None:
+    playbook, identity, receipt_input, trust_receipt = _learning_receipt_fixture()
+    trust_receipt[field] = wrong_value
+
+    assert not _matches(playbook, identity, receipt_input, trust_receipt)