fix(security): reconcile asset control plane truth

apps/api/tests/test_asset_inventory_database_type_migration.py

@@ -0,0 +1,75 @@
+import textwrap
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+FORWARD = (
+    ROOT
+    / "apps"
+    / "api"
+    / "migrations"
+    / "asset_inventory_database_types_2026-07-14.sql"
+)
+ROLLBACK = (
+    ROOT
+    / "apps"
+    / "api"
+    / "migrations"
+    / "asset_inventory_database_types_2026-07-14_down.sql"
+)
+CD_WORKFLOW = ROOT / ".gitea" / "workflows" / "cd.yaml"
+
+
+def test_forward_migration_repairs_type_constraint_without_row_deletion() -> None:
+    sql = FORWARD.read_text(encoding="utf-8")
+    upper = sql.upper()
+
+    assert "SET lock_timeout = '5s'" in sql
+    assert "SET statement_timeout = '30s'" in sql
+    assert "'database','table'" in sql
+    assert "DROP CONSTRAINT IF EXISTS asset_inventory_type_valid" in sql
+    assert "NOT VALID" in sql
+    assert "VALIDATE CONSTRAINT asset_inventory_type_valid" in sql
+    assert "gitea_cd_asset_schema_reconciler" in sql
+    assert '"work_item_id":"AIA-P0-006"' in sql
+    assert "automation_operation_log" in sql
+    assert "DELETE FROM" not in upper
+    assert "DROP TABLE" not in upper
+    assert "TRUNCATE" not in upper
+
+
+def test_rollback_refuses_to_remove_types_while_assets_depend_on_them() -> None:
+    sql = ROLLBACK.read_text(encoding="utf-8")
+    upper = sql.upper()
+
+    assert "asset_type IN ('database', 'table')" in sql
+    assert "IF dependent_rows > 0" in sql
+    assert "RAISE EXCEPTION" in sql
+    assert "DELETE FROM" not in upper
+    assert "TRUNCATE" not in upper
+
+
+def test_cd_applies_and_independently_verifies_the_bounded_migration() -> None:
+    workflow = CD_WORKFLOW.read_text(encoding="utf-8")
+    step = workflow.split(
+        "- name: Reconcile Asset Inventory Type Constraint", 1
+    )[1].split("# 2026-03-31 ogt: 移除中間通知", 1)[0]
+    script = step.split("run: |", 1)[1]
+
+    assert "MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}" in step
+    assert "DATABASE_URL: ${{ secrets.DATABASE_URL }}" in step
+    assert "${{ secrets." not in script
+    assert "asset_inventory_database_types_2026-07-14.sql" in script
+    assert "asset_inventory_type_migration_applied=" in script
+    assert "database_type_allowed=" in script
+    assert "table_type_allowed=" in script
+    assert "constraint_type_set_exact=" in script
+    assert "constraint_validated=" in script
+    assert "durable_receipt_present=" in script
+    assert "asset_inventory_type_constraint_verifier_failed" in script
+
+
+def test_cd_embedded_migration_python_compiles() -> None:
+    workflow = CD_WORKFLOW.read_text(encoding="utf-8")
+    source = workflow.split("python - <<'PY'", 1)[1].split("          PY", 1)[0]
+
+    compile(textwrap.dedent(source), "<asset-inventory-migration>", "exec")

apps/api/tests/test_asset_scanner_job.py

@@ -6,6 +6,8 @@ from src.jobs import asset_scanner_job
 
 
 class _Result:
+    rowcount = 1
+
     def scalar(self):
         return "00000000-0000-0000-0000-000000000001"
 
@@ -41,15 +43,6 @@ def test_background_scanner_scopes_every_database_operation(monkeypatch) -> None
         )
         assert run_id == "00000000-0000-0000-0000-000000000001"
 
-        await asset_scanner_job._finish_discovery_run(
-            run_id=run_id,
-            status="success",
-            total_assets=1,
-            new_assets=1,
-            modified_assets=0,
-            duration_ms=10,
-            error=None,
-        )
         inserted, modified = await asset_scanner_job._upsert_assets(
             [
                 {
@@ -66,6 +59,17 @@ def test_background_scanner_scopes_every_database_operation(monkeypatch) -> None
         )
         assert (inserted, modified) == (1, 0)
 
+        disappeared = await asset_scanner_job._deprecate_missing_assets(
+            [
+                {
+                    "asset_key": "host/test",
+                }
+            ],
+            ("host/",),
+            run_id,
+        )
+        assert disappeared == 1
+
         written = await asset_scanner_job._upsert_relationships(
             [
                 {
@@ -78,14 +82,15 @@ def test_background_scanner_scopes_every_database_operation(monkeypatch) -> None
         assert written == 1
 
         await asset_scanner_job._write_coverage_snapshots(run_id)
-        await asset_scanner_job._log_aol_asset_discovered(
+        await asset_scanner_job._finalize_discovery_run(
             run_id=run_id,
             triggered_by="cron",
-            total=1,
-            new=1,
-            modified=0,
-            duration_ms=10,
             status="success",
+            total_assets=1,
+            new_assets=1,
+            modified_assets=0,
+            disappeared_assets=1,
+            duration_ms=10,
             error=None,
             scope=["k8s", "prometheus", "database_catalog"],
             scan_depth="shallow",
@@ -96,6 +101,183 @@ def test_background_scanner_scopes_every_database_operation(monkeypatch) -> None
     assert requested_projects == ["awoooi"] * 6
 
 
+def test_asset_upsert_propagates_database_failure(monkeypatch) -> None:
+    class FailingDatabase:
+        async def execute(self, statement, parameters=None):
+            raise RuntimeError("constraint rejected")
+
+    class FailingContext:
+        async def __aenter__(self):
+            return FailingDatabase()
+
+        async def __aexit__(self, exc_type, exc, traceback):
+            return False
+
+    monkeypatch.setattr(
+        "src.db.base.get_db_context",
+        lambda project_id=None: FailingContext(),
+    )
+
+    async def exercise() -> None:
+        try:
+            await asset_scanner_job._upsert_assets(
+                [
+                    {
+                        "asset_key": "postgres/database/awoooi",
+                        "asset_type": "database",
+                        "host": None,
+                        "namespace": None,
+                        "name": "awoooi",
+                        "metadata": {},
+                        "tags": ["source:test"],
+                    }
+                ],
+                "00000000-0000-0000-0000-000000000001",
+            )
+        except RuntimeError as exc:
+            assert str(exc) == "constraint rejected"
+        else:
+            raise AssertionError("asset upsert failure must propagate")
+
+    asyncio.run(exercise())
+
+
+def test_deprecate_missing_assets_is_bounded_to_successful_prefixes(monkeypatch) -> None:
+    captured: dict[str, object] = {}
+
+    class CapturingDatabase:
+        async def execute(self, statement, parameters=None):
+            captured["sql"] = str(statement)
+            captured["parameters"] = parameters
+            return _Result()
+
+    class CapturingContext:
+        async def __aenter__(self):
+            return CapturingDatabase()
+
+        async def __aexit__(self, exc_type, exc, traceback):
+            return False
+
+    monkeypatch.setattr(
+        "src.db.base.get_db_context",
+        lambda project_id=None: CapturingContext(),
+    )
+
+    disappeared = asyncio.run(
+        asset_scanner_job._deprecate_missing_assets(
+            [
+                {"asset_key": "k8s/pod/prod/current"},
+                {"asset_key": "prometheus_target/node/192.168.0.110:9100"},
+            ],
+            ("k8s/pod/",),
+            "00000000-0000-0000-0000-000000000001",
+        )
+    )
+
+    assert disappeared == 1
+    assert "last_seen_at <" in str(captured["sql"])
+    assert captured["parameters"] == {
+        "patterns": ["k8s/pod/%"],
+        "seen_keys": ["k8s/pod/prod/current"],
+        "rid": "00000000-0000-0000-0000-000000000001",
+    }
+
+
+def test_runtime_collection_reports_failed_sources_without_reconciling_them(
+    monkeypatch,
+) -> None:
+    async def fake_kubectl(resource: str, all_namespaces: bool = True):
+        if resource == "nodes":
+            raise RuntimeError("nodes unavailable")
+        return {"items": []}
+
+    async def fake_prometheus():
+        return [], []
+
+    async def fake_database_catalog():
+        return [], []
+
+    monkeypatch.setattr(asset_scanner_job, "_fetch_kubectl_json", fake_kubectl)
+    monkeypatch.setattr(
+        asset_scanner_job,
+        "_collect_prometheus_targets",
+        fake_prometheus,
+    )
+    monkeypatch.setattr(
+        asset_scanner_job,
+        "_collect_database_catalog_assets",
+        fake_database_catalog,
+    )
+
+    assets, relationships, prefixes, errors = asyncio.run(
+        asset_scanner_job._collect_runtime_assets()
+    )
+
+    assert assets == []
+    assert relationships == []
+    assert "k8s/node/" not in prefixes
+    assert "k8s/pod/" in prefixes
+    assert "prometheus_target/" in prefixes
+    assert errors == ("k8s_nodes",)
+
+
+def test_scan_terminal_is_failed_when_any_collector_is_incomplete(monkeypatch) -> None:
+    terminal: dict[str, object] = {}
+
+    async def fake_start(*args, **kwargs):
+        return "00000000-0000-0000-0000-000000000001"
+
+    async def fake_collect():
+        return [], [], (), ("prometheus_targets",)
+
+    async def fake_upsert(*args, **kwargs):
+        return 0, 0
+
+    async def fake_deprecate(*args, **kwargs):
+        return 0
+
+    async def fake_relationships(*args, **kwargs):
+        return 0
+
+    async def fake_coverage(*args, **kwargs):
+        return None
+
+    async def fake_finalize(**kwargs):
+        terminal.update(kwargs)
+
+    monkeypatch.setattr(asset_scanner_job, "_start_discovery_run", fake_start)
+    monkeypatch.setattr(asset_scanner_job, "_collect_runtime_assets", fake_collect)
+    monkeypatch.setattr(asset_scanner_job, "_upsert_assets", fake_upsert)
+    monkeypatch.setattr(
+        asset_scanner_job,
+        "_deprecate_missing_assets",
+        fake_deprecate,
+    )
+    monkeypatch.setattr(
+        asset_scanner_job,
+        "_upsert_relationships",
+        fake_relationships,
+    )
+    monkeypatch.setattr(
+        asset_scanner_job,
+        "_write_coverage_snapshots",
+        fake_coverage,
+    )
+    monkeypatch.setattr(
+        asset_scanner_job,
+        "_finalize_discovery_run",
+        fake_finalize,
+    )
+
+    run_id = asyncio.run(asset_scanner_job.scan_once())
+
+    assert run_id == "00000000-0000-0000-0000-000000000001"
+    assert terminal["status"] == "failed"
+    assert terminal["error"] == (
+        "RuntimeError: asset_collector_failures=prometheus_targets"
+    )
+
+
 def test_database_catalog_collector_reads_metadata_only(monkeypatch) -> None:
     requested_projects: list[str | None] = []