fix(security): reconcile asset control plane truth
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m27s
CD Pipeline / build-and-deploy (push) Failing after 3m57s
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
ogt
2026-07-14 10:47:33 +08:00
parent 467542a018
commit b6ddee7984
9 changed files with 869 additions and 170 deletions

View File

@@ -0,0 +1,40 @@
-- Reconcile the production asset_inventory type constraint with source truth.
-- Safety: allowlist-only DDL, bounded locks, no row data reads or deletes.
SET lock_timeout = '5s';
SET statement_timeout = '30s';
ALTER TABLE asset_inventory
DROP CONSTRAINT IF EXISTS asset_inventory_type_valid;
ALTER TABLE asset_inventory
ADD CONSTRAINT asset_inventory_type_valid CHECK (asset_type IN (
'host','container','k8s_workload','k8s_resource','database','table',
'website','api_endpoint','package','log_stream','km_entry',
'frontend','backend','ci_pipeline','gitea_repo','monitoring_target',
'secret','volume','network','certificate','scheduled_job',
'message_queue','cache','dashboard','ai_agent','llm_model',
'third_party_service','backup_target'
)) NOT VALID;
ALTER TABLE asset_inventory
VALIDATE CONSTRAINT asset_inventory_type_valid;
INSERT INTO automation_operation_log (
operation_type,
actor,
status,
input,
output,
tags
) VALUES (
'asset_discovered',
'gitea_cd_asset_schema_reconciler',
'success',
'{"work_item_id":"AIA-P0-006","migration":"asset_inventory_database_types_2026-07-14"}'::jsonb,
'{"constraint_validated":true,"constraint_type_set_exact":true,"database_type_allowed":true,"table_type_allowed":true}'::jsonb,
ARRAY['asset_control_plane','schema_migration','controlled_apply','post_verified']
);
COMMENT ON CONSTRAINT asset_inventory_type_valid ON asset_inventory IS
'Asset control-plane allowlist reconciled by Gitea CD; includes database and table metadata assets.';

View File

@@ -0,0 +1,57 @@
-- Roll back database/table asset types only when no such inventory rows exist.
-- This script never deletes rows to make rollback possible.
SET lock_timeout = '5s';
SET statement_timeout = '30s';
DO $migration$
DECLARE
dependent_rows BIGINT;
BEGIN
SELECT COUNT(*)
INTO dependent_rows
FROM asset_inventory
WHERE asset_type IN ('database', 'table');
IF dependent_rows > 0 THEN
RAISE EXCEPTION
'rollback blocked: % database/table asset rows must be reconciled first',
dependent_rows;
END IF;
ALTER TABLE asset_inventory
DROP CONSTRAINT IF EXISTS asset_inventory_type_valid;
ALTER TABLE asset_inventory
ADD CONSTRAINT asset_inventory_type_valid CHECK (asset_type IN (
'host','container','k8s_workload','k8s_resource',
'website','api_endpoint','package','log_stream','km_entry',
'frontend','backend','ci_pipeline','gitea_repo','monitoring_target',
'secret','volume','network','certificate','scheduled_job',
'message_queue','cache','dashboard','ai_agent','llm_model',
'third_party_service','backup_target'
)) NOT VALID;
ALTER TABLE asset_inventory
VALIDATE CONSTRAINT asset_inventory_type_valid;
INSERT INTO automation_operation_log (
operation_type,
actor,
status,
input,
output,
tags
) VALUES (
'asset_discovered',
'gitea_cd_asset_schema_reconciler',
'rolled_back',
'{"work_item_id":"AIA-P0-006","migration":"asset_inventory_database_types_2026-07-14_down"}'::jsonb,
'{"database_type_allowed":false,"table_type_allowed":false,"dependent_rows":0}'::jsonb,
ARRAY['asset_control_plane','schema_migration','controlled_rollback']
);
END
$migration$;
COMMENT ON CONSTRAINT asset_inventory_type_valid ON asset_inventory IS
'Rollback allowlist without database/table types; blocked while dependent asset rows exist.';

View File

@@ -92,7 +92,7 @@ async def scan_once(
triggered_by: str = "cron",
scope: tuple[str, ...] = ("k8s", "prometheus", "database_catalog"),
scan_depth: str = "shallow",
) -> str | None:
) -> str:
"""
執行一次資產盤點。
@@ -102,15 +102,14 @@ async def scan_once(
scan_depth: shallow (僅 list) / deep (含 describe) / full
Returns:
run_id (UUID 字串) 或 None (寫 header 失敗時)
run_id (UUID 字串)。無法建立 durable header 時會拋出例外。
"""
started_ms = _time.time()
run_id = await _start_discovery_run(triggered_by, list(scope), scan_depth)
if not run_id:
return None
new_count = 0
modified_count = 0
disappeared_count = 0
total_count = 0
error_msg: str | None = None
@@ -119,11 +118,29 @@ async def scan_once(
# 資源類型: pods (container), deployments/statefulsets/daemonsets (k8s_workload),
# services (k8s_resource), nodes (host), configmaps (k8s_resource)
# 跳過: secrets (awoooi-executor RBAC 不允許 list)
runtime_assets, relationships = await _collect_runtime_assets()
(
runtime_assets,
relationships,
reconciled_prefixes,
collector_errors,
) = await _collect_runtime_assets()
total_count = len(runtime_assets)
# UPSERT inventory
new_count, modified_count = await _upsert_assets(runtime_assets, run_id)
if new_count + modified_count != total_count:
raise RuntimeError(
"asset_upsert_count_mismatch "
f"expected={total_count} persisted={new_count + modified_count}"
)
# Only reconcile sources that completed collection in this run. This
# prevents a failed/empty collector from deprecating healthy assets.
disappeared_count = await _deprecate_missing_assets(
runtime_assets,
reconciled_prefixes,
run_id,
)
# 建立 asset_relationship (OwnerReference + Service selector + Pod volumes)
rel_written = await _upsert_relationships(relationships)
@@ -132,6 +149,10 @@ async def scan_once(
await _write_coverage_snapshots(run_id)
logger.info("asset_scan_relationships_written", run_id=run_id, relationships=rel_written)
if collector_errors:
raise RuntimeError(
"asset_collector_failures=" + ",".join(collector_errors)
)
except Exception as e:
error_msg = f"{type(e).__name__}: {e}"[:1000]
@@ -140,26 +161,18 @@ async def scan_once(
duration_ms = int((_time.time() - started_ms) * 1000)
final_status = "failed" if error_msg else "success"
await _finish_discovery_run(
# The discovery terminal and AOL receipt share one DB transaction. A run
# cannot be marked successful without its durable audit receipt.
await _finalize_discovery_run(
run_id=run_id,
triggered_by=triggered_by,
status=final_status,
total_assets=total_count,
new_assets=new_count,
modified_assets=modified_count,
disappeared_assets=disappeared_count,
duration_ms=duration_ms,
error=error_msg,
)
# ADR-090 § aol 留痕 — asset_discovered 是合法 op_type
await _log_aol_asset_discovered(
run_id=run_id,
triggered_by=triggered_by,
total=total_count,
new=new_count,
modified=modified_count,
duration_ms=duration_ms,
status=final_status,
error=error_msg,
scope=list(scope),
scan_depth=scan_depth,
)
@@ -171,6 +184,7 @@ async def scan_once(
total=total_count,
new=new_count,
modified=modified_count,
disappeared=disappeared_count,
duration_ms=duration_ms,
)
return run_id
@@ -345,7 +359,12 @@ def _build_configmap_asset(item: dict[str, Any]) -> dict[str, Any]:
}
async def _collect_runtime_assets() -> tuple[list[dict[str, Any]], list[dict[str, str]]]:
async def _collect_runtime_assets() -> tuple[
list[dict[str, Any]],
list[dict[str, str]],
tuple[str, ...],
tuple[str, ...],
]:
"""
掃多種 K8s 資源類型 + 提取 relationship.
@@ -360,10 +379,16 @@ async def _collect_runtime_assets() -> tuple[list[dict[str, Any]], list[dict[str
原 code 跳過 ReplicaSet → Pod→Deployment 全部漏掉.
修: 先掃 ReplicaSet 建 rs_to_deployment map,Pod 用 rs_name 反查 Deployment.
回傳 (assets, relationships) tuple.
回傳 (assets, relationships, reconciled_asset_key_prefixes, collector_errors) tuple.
Only prefixes whose source query completed successfully are returned. The
caller may safely deprecate missing assets within those prefixes without
treating collector failures as asset deletion.
"""
assets: list[dict[str, Any]] = []
relationships: list[dict[str, str]] = []
reconciled_prefixes: set[str] = set()
collector_errors: list[str] = []
# 0. ReplicaSets — 僅作為 Pod→Deployment 橋樑,不寫入 asset_inventory
rs_to_deployment: dict[str, str] = {} # "ns/rs_name" -> "deployment_name"
@@ -378,14 +403,17 @@ async def _collect_runtime_assets() -> tuple[list[dict[str, Any]], list[dict[str
rs_to_deployment[f"{rs_ns}/{rs_name}"] = ref.get("name", "")
except Exception as e:
logger.warning("collect_replicasets_failed", error=str(e))
collector_errors.append("k8s_replicasets")
# 1. Nodes (不帶 ns)
try:
payload = await _fetch_kubectl_json("nodes", all_namespaces=False)
for item in payload.get("items", []) or []:
assets.append(_build_node_asset(item))
reconciled_prefixes.add("k8s/node/")
except Exception as e:
logger.warning("collect_nodes_failed", error=str(e))
collector_errors.append("k8s_nodes")
# 2. Pods — 主體 + 從 ownerReferences 建 relationship
pod_by_key: dict[str, dict[str, Any]] = {}
@@ -436,8 +464,10 @@ async def _collect_runtime_assets() -> tuple[list[dict[str, Any]], list[dict[str
"to_key": f"k8s/configmap/{ns}/{cm}",
"relationship_type": "depends_on",
})
reconciled_prefixes.add("k8s/pod/")
except Exception as e:
logger.warning("collect_pods_failed", error=str(e))
collector_errors.append("k8s_pods")
# 3. Deployments / StatefulSets / DaemonSets
for kind, resource in (("Deployment", "deployments"), ("StatefulSet", "statefulsets"), ("DaemonSet", "daemonsets")):
@@ -445,8 +475,10 @@ async def _collect_runtime_assets() -> tuple[list[dict[str, Any]], list[dict[str
payload = await _fetch_kubectl_json(resource)
for item in payload.get("items", []) or []:
assets.append(_build_workload_asset(item, kind))
reconciled_prefixes.add(f"k8s/{kind.lower()}/")
except Exception as e:
logger.warning(f"collect_{resource}_failed", error=str(e))
collector_errors.append(f"k8s_{resource}")
# 4. Services + routes_to Pod (via selector match)
try:
@@ -471,16 +503,20 @@ async def _collect_runtime_assets() -> tuple[list[dict[str, Any]], list[dict[str
"to_key": pod_key,
"relationship_type": "routes_to",
})
reconciled_prefixes.add("k8s/service/")
except Exception as e:
logger.warning("collect_services_failed", error=str(e))
collector_errors.append("k8s_services")
# 5. ConfigMaps
try:
payload = await _fetch_kubectl_json("configmaps")
for item in payload.get("items", []) or []:
assets.append(_build_configmap_asset(item))
reconciled_prefixes.add("k8s/configmap/")
except Exception as e:
logger.warning("collect_configmaps_failed", error=str(e))
collector_errors.append("k8s_configmaps")
# 6. Prometheus targets — 補齊 host-install services (110/112/188/125 等非 K8s)
# Gap 1 修補 (2026-04-19 audit): 原本 asset_inventory 只涵蓋 K8s,
@@ -490,18 +526,30 @@ async def _collect_runtime_assets() -> tuple[list[dict[str, Any]], list[dict[str
prom_assets, host_relationships = await _collect_prometheus_targets()
assets.extend(prom_assets)
relationships.extend(host_relationships)
reconciled_prefixes.update(("host/", "prometheus_target/"))
except Exception as e:
logger.warning("collect_prometheus_targets_failed", error=str(e))
collector_errors.append("prometheus_targets")
# 7. PostgreSQL catalog — database/table/view metadata only, never row data.
try:
database_assets, database_relationships = await _collect_database_catalog_assets()
assets.extend(database_assets)
relationships.extend(database_relationships)
if database_assets:
reconciled_prefixes.update(
("postgres/database/", "postgres/relation/")
)
except Exception as e:
logger.warning("collect_database_catalog_failed", error=str(e))
collector_errors.append("database_catalog")
return assets, relationships
return (
assets,
relationships,
tuple(sorted(reconciled_prefixes)),
tuple(collector_errors),
)
def _is_valid_ipv4(s: str) -> bool:
@@ -543,14 +591,10 @@ async def _collect_prometheus_targets() -> tuple[list[dict[str, Any]], list[dict
seen_hosts: set[str] = set()
url = f"{settings.PROMETHEUS_URL.rstrip('/')}/api/v1/targets"
try:
async with httpx.AsyncClient(timeout=10.0, trust_env=False) as client:
resp = await client.get(url, params={"state": "active"})
resp.raise_for_status()
data = resp.json()
except Exception as e:
logger.warning("prometheus_targets_fetch_failed", error=str(e))
return assets, relationships
async with httpx.AsyncClient(timeout=10.0, trust_env=False) as client:
resp = await client.get(url, params={"state": "active"})
resp.raise_for_status()
data = resp.json()
for t in (data.get("data", {}) or {}).get("activeTargets", []) or []:
labels = t.get("labels", {}) or {}
@@ -744,11 +788,11 @@ async def _start_discovery_run(
triggered_by: str,
scope: list[str],
scan_depth: str,
) -> str | None:
) -> str:
"""
寫 asset_discovery_run header (status='running'), 回傳 run_id。
失敗 → log warning + 回 None,主流程靜默跳過本次 scan
失敗會向排程器傳播並進入 bounded backoff不可靜默跳過整輪掃描
"""
try:
from sqlalchemy import text as _sql
@@ -783,51 +827,97 @@ async def _start_discovery_run(
},
)
run_id = row.scalar()
return str(run_id) if run_id else None
if not run_id:
raise RuntimeError("asset_discovery_run_missing_run_id")
return str(run_id)
except Exception as e:
logger.warning("asset_discovery_run_start_failed", error=str(e))
return None
raise
async def _finish_discovery_run(
async def _finalize_discovery_run(
run_id: str,
triggered_by: str,
status: str,
total_assets: int,
new_assets: int,
modified_assets: int,
disappeared_assets: int,
duration_ms: int,
error: str | None,
scope: list[str],
scan_depth: str,
) -> None:
try:
from sqlalchemy import text as _sql
"""Atomically persist the discovery terminal and its AOL receipt."""
from sqlalchemy import text as _sql
from src.db.base import get_db_context
from src.db.base import get_db_context
async with get_db_context(_PROJECT_ID) as db:
await db.execute(
_sql("""
UPDATE asset_discovery_run
SET status = :st,
ended_at = NOW(),
total_assets = :total,
new_assets = :new,
modified_assets = :mod,
duration_ms = :dur,
error = :err
WHERE run_id = CAST(:rid AS uuid)
"""),
{
"st": status,
"total": total_assets,
"new": new_assets,
"mod": modified_assets,
"dur": duration_ms,
"err": error,
"rid": run_id,
},
)
except Exception as e:
logger.warning("asset_discovery_run_finish_failed", run_id=run_id, error=str(e))
input_payload = {
"run_id": run_id,
"triggered_by": triggered_by,
"scope": scope,
"scan_depth": scan_depth,
}
output_payload = {
"run_id": run_id,
"total_assets": total_assets,
"new_assets": new_assets,
"modified_assets": modified_assets,
"disappeared_assets": disappeared_assets,
}
async with get_db_context(_PROJECT_ID) as db:
await db.execute(
_sql("""
UPDATE asset_discovery_run
SET status = :st,
ended_at = NOW(),
total_assets = :total,
new_assets = :new,
modified_assets = :mod,
disappeared_assets = :gone,
duration_ms = :dur,
error = :err
WHERE run_id = CAST(:rid AS uuid)
"""),
{
"st": status,
"total": total_assets,
"new": new_assets,
"mod": modified_assets,
"gone": disappeared_assets,
"dur": duration_ms,
"err": error,
"rid": run_id,
},
)
await db.execute(
_sql("""
INSERT INTO automation_operation_log (
operation_type, actor, status,
input, output, run_id, duration_ms, error, tags
) VALUES (
'asset_discovered',
'asset_scanner',
:st,
CAST(:input AS jsonb),
CAST(:output AS jsonb),
CAST(:rid AS uuid),
:dur, :err,
:tags
)
"""),
{
"st": "success" if status == "success" else "failed",
"input": _json.dumps(input_payload, ensure_ascii=False),
"output": _json.dumps(output_payload, ensure_ascii=False),
"rid": run_id,
"dur": duration_ms,
"err": (error or "")[:2000] if error else None,
"tags": ["asset_scanner", "discovery", *scope],
},
)
async def _upsert_assets(
@@ -891,10 +981,62 @@ async def _upsert_assets(
modified_count += 1
except Exception as e:
logger.exception("asset_upsert_failed", run_id=run_id, error=str(e))
raise
return new_count, modified_count
async def _deprecate_missing_assets(
assets: list[dict[str, Any]],
reconciled_prefixes: tuple[str, ...],
run_id: str,
) -> int:
"""Deprecate assets missing from collectors that succeeded in this run."""
prefixes = sorted({prefix for prefix in reconciled_prefixes if prefix})
if not prefixes:
return 0
seen_keys = sorted(
{
str(asset["asset_key"])
for asset in assets
if any(str(asset["asset_key"]).startswith(prefix) for prefix in prefixes)
}
)
from sqlalchemy import text as _sql
from src.db.base import get_db_context
async with get_db_context(_PROJECT_ID) as db:
result = await db.execute(
_sql(
"""
UPDATE asset_inventory
SET lifecycle_state = 'deprecated',
decommissioned_at = NOW(),
updated_at = NOW()
WHERE lifecycle_state = 'active'
AND asset_key LIKE ANY(CAST(:patterns AS text[]))
AND last_seen_at < (
SELECT started_at
FROM asset_discovery_run
WHERE run_id = CAST(:rid AS uuid)
)
AND NOT (asset_key = ANY(CAST(:seen_keys AS text[])))
"""
),
{
"patterns": [f"{prefix}%" for prefix in prefixes],
"seen_keys": seen_keys,
"rid": run_id,
},
)
rowcount = getattr(result, "rowcount", 0) or 0
return max(int(rowcount), 0)
async def _upsert_relationships(relationships: list[dict[str, str]]) -> int:
"""
UPSERT asset_relationship (from_asset_id/to_asset_id/relationship_type 為 UNIQUE).
@@ -939,8 +1081,10 @@ async def _upsert_relationships(relationships: list[dict[str, str]]) -> int:
except Exception as e:
logger.debug("relationship_upsert_skipped",
from_key=rel["from_key"], to_key=rel["to_key"], error=str(e))
raise
except Exception as e:
logger.warning("relationship_upsert_failed", error=str(e))
raise
return written
@@ -950,99 +1094,33 @@ async def _write_coverage_snapshots(run_id: str) -> None:
後續 service (rule_catalog / playbook_extractor / km_writer) 會 UPDATE 對應維度。
"""
try:
from sqlalchemy import text as _sql
from sqlalchemy import text as _sql
from src.db.base import get_db_context
from src.db.base import get_db_context
async with get_db_context(_PROJECT_ID) as db:
# 一次性 INSERT: 取所有 active asset × 7 dimensions
await db.execute(
_sql("""
INSERT INTO asset_coverage_snapshot (
run_id, asset_id, dimension, coverage_status,
evidence, detected_by
)
SELECT
CAST(:rid AS uuid),
ai.asset_id,
d.dimension,
'unknown' AS coverage_status,
'{}'::jsonb,
'asset_scanner' AS detected_by
FROM asset_inventory ai
CROSS JOIN (
VALUES ('auto_monitoring'),('auto_alerting'),('auto_rule_creation'),
('auto_rule_matching'),('auto_playbook'),('auto_remediation'),
('auto_km_creation')
) AS d(dimension)
WHERE ai.lifecycle_state = 'active'
ON CONFLICT (run_id, asset_id, dimension) DO NOTHING
"""),
{"rid": run_id},
)
except Exception as e:
logger.warning("asset_coverage_write_failed", run_id=run_id, error=str(e))
async def _log_aol_asset_discovered(
run_id: str,
triggered_by: str,
total: int,
new: int,
modified: int,
duration_ms: int,
status: str,
error: str | None,
scope: list[str],
scan_depth: str,
) -> None:
"""寫 automation_operation_log(asset_discovered)。失敗只 log 不阻塞。"""
try:
from sqlalchemy import text as _sql
from src.db.base import get_db_context
aol_status = "success" if status == "success" else "failed"
input_payload = {
"run_id": run_id,
"triggered_by": triggered_by,
"scope": scope,
"scan_depth": scan_depth,
}
output_payload = {
"run_id": run_id,
"total_assets": total,
"new_assets": new,
"modified_assets": modified,
}
async with get_db_context(_PROJECT_ID) as db:
await db.execute(
_sql("""
INSERT INTO automation_operation_log (
operation_type, actor, status,
input, output, run_id, duration_ms, error, tags
) VALUES (
'asset_discovered',
'asset_scanner',
:st,
CAST(:input AS jsonb),
CAST(:output AS jsonb),
CAST(:rid AS uuid),
:dur, :err,
:tags
)
"""),
{
"st": aol_status,
"input": _json.dumps(input_payload, ensure_ascii=False),
"output": _json.dumps(output_payload, ensure_ascii=False),
"rid": run_id,
"dur": duration_ms,
"err": (error or "")[:2000] if error else None,
"tags": ["asset_scanner", "discovery", *scope],
},
)
except Exception as e:
logger.warning("asset_scanner_aol_write_failed", run_id=run_id, error=str(e))
async with get_db_context(_PROJECT_ID) as db:
# 一次性 INSERT: 取所有 active asset × 7 dimensions
await db.execute(
_sql("""
INSERT INTO asset_coverage_snapshot (
run_id, asset_id, dimension, coverage_status,
evidence, detected_by
)
SELECT
CAST(:rid AS uuid),
ai.asset_id,
d.dimension,
'unknown' AS coverage_status,
'{}'::jsonb,
'asset_scanner' AS detected_by
FROM asset_inventory ai
CROSS JOIN (
VALUES ('auto_monitoring'),('auto_alerting'),('auto_rule_creation'),
('auto_rule_matching'),('auto_playbook'),('auto_remediation'),
('auto_km_creation')
) AS d(dimension)
WHERE ai.lifecycle_state = 'active'
ON CONFLICT (run_id, asset_id, dimension) DO NOTHING
"""),
{"rid": run_id},
)

View File

@@ -0,0 +1,75 @@
import textwrap
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
FORWARD = (
ROOT
/ "apps"
/ "api"
/ "migrations"
/ "asset_inventory_database_types_2026-07-14.sql"
)
ROLLBACK = (
ROOT
/ "apps"
/ "api"
/ "migrations"
/ "asset_inventory_database_types_2026-07-14_down.sql"
)
CD_WORKFLOW = ROOT / ".gitea" / "workflows" / "cd.yaml"
def test_forward_migration_repairs_type_constraint_without_row_deletion() -> None:
sql = FORWARD.read_text(encoding="utf-8")
upper = sql.upper()
assert "SET lock_timeout = '5s'" in sql
assert "SET statement_timeout = '30s'" in sql
assert "'database','table'" in sql
assert "DROP CONSTRAINT IF EXISTS asset_inventory_type_valid" in sql
assert "NOT VALID" in sql
assert "VALIDATE CONSTRAINT asset_inventory_type_valid" in sql
assert "gitea_cd_asset_schema_reconciler" in sql
assert '"work_item_id":"AIA-P0-006"' in sql
assert "automation_operation_log" in sql
assert "DELETE FROM" not in upper
assert "DROP TABLE" not in upper
assert "TRUNCATE" not in upper
def test_rollback_refuses_to_remove_types_while_assets_depend_on_them() -> None:
sql = ROLLBACK.read_text(encoding="utf-8")
upper = sql.upper()
assert "asset_type IN ('database', 'table')" in sql
assert "IF dependent_rows > 0" in sql
assert "RAISE EXCEPTION" in sql
assert "DELETE FROM" not in upper
assert "TRUNCATE" not in upper
def test_cd_applies_and_independently_verifies_the_bounded_migration() -> None:
workflow = CD_WORKFLOW.read_text(encoding="utf-8")
step = workflow.split(
"- name: Reconcile Asset Inventory Type Constraint", 1
)[1].split("# 2026-03-31 ogt: 移除中間通知", 1)[0]
script = step.split("run: |", 1)[1]
assert "MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}" in step
assert "DATABASE_URL: ${{ secrets.DATABASE_URL }}" in step
assert "${{ secrets." not in script
assert "asset_inventory_database_types_2026-07-14.sql" in script
assert "asset_inventory_type_migration_applied=" in script
assert "database_type_allowed=" in script
assert "table_type_allowed=" in script
assert "constraint_type_set_exact=" in script
assert "constraint_validated=" in script
assert "durable_receipt_present=" in script
assert "asset_inventory_type_constraint_verifier_failed" in script
def test_cd_embedded_migration_python_compiles() -> None:
workflow = CD_WORKFLOW.read_text(encoding="utf-8")
source = workflow.split("python - <<'PY'", 1)[1].split(" PY", 1)[0]
compile(textwrap.dedent(source), "<asset-inventory-migration>", "exec")

View File

@@ -6,6 +6,8 @@ from src.jobs import asset_scanner_job
class _Result:
rowcount = 1
def scalar(self):
return "00000000-0000-0000-0000-000000000001"
@@ -41,15 +43,6 @@ def test_background_scanner_scopes_every_database_operation(monkeypatch) -> None
)
assert run_id == "00000000-0000-0000-0000-000000000001"
await asset_scanner_job._finish_discovery_run(
run_id=run_id,
status="success",
total_assets=1,
new_assets=1,
modified_assets=0,
duration_ms=10,
error=None,
)
inserted, modified = await asset_scanner_job._upsert_assets(
[
{
@@ -66,6 +59,17 @@ def test_background_scanner_scopes_every_database_operation(monkeypatch) -> None
)
assert (inserted, modified) == (1, 0)
disappeared = await asset_scanner_job._deprecate_missing_assets(
[
{
"asset_key": "host/test",
}
],
("host/",),
run_id,
)
assert disappeared == 1
written = await asset_scanner_job._upsert_relationships(
[
{
@@ -78,14 +82,15 @@ def test_background_scanner_scopes_every_database_operation(monkeypatch) -> None
assert written == 1
await asset_scanner_job._write_coverage_snapshots(run_id)
await asset_scanner_job._log_aol_asset_discovered(
await asset_scanner_job._finalize_discovery_run(
run_id=run_id,
triggered_by="cron",
total=1,
new=1,
modified=0,
duration_ms=10,
status="success",
total_assets=1,
new_assets=1,
modified_assets=0,
disappeared_assets=1,
duration_ms=10,
error=None,
scope=["k8s", "prometheus", "database_catalog"],
scan_depth="shallow",
@@ -96,6 +101,183 @@ def test_background_scanner_scopes_every_database_operation(monkeypatch) -> None
assert requested_projects == ["awoooi"] * 6
def test_asset_upsert_propagates_database_failure(monkeypatch) -> None:
class FailingDatabase:
async def execute(self, statement, parameters=None):
raise RuntimeError("constraint rejected")
class FailingContext:
async def __aenter__(self):
return FailingDatabase()
async def __aexit__(self, exc_type, exc, traceback):
return False
monkeypatch.setattr(
"src.db.base.get_db_context",
lambda project_id=None: FailingContext(),
)
async def exercise() -> None:
try:
await asset_scanner_job._upsert_assets(
[
{
"asset_key": "postgres/database/awoooi",
"asset_type": "database",
"host": None,
"namespace": None,
"name": "awoooi",
"metadata": {},
"tags": ["source:test"],
}
],
"00000000-0000-0000-0000-000000000001",
)
except RuntimeError as exc:
assert str(exc) == "constraint rejected"
else:
raise AssertionError("asset upsert failure must propagate")
asyncio.run(exercise())
def test_deprecate_missing_assets_is_bounded_to_successful_prefixes(monkeypatch) -> None:
captured: dict[str, object] = {}
class CapturingDatabase:
async def execute(self, statement, parameters=None):
captured["sql"] = str(statement)
captured["parameters"] = parameters
return _Result()
class CapturingContext:
async def __aenter__(self):
return CapturingDatabase()
async def __aexit__(self, exc_type, exc, traceback):
return False
monkeypatch.setattr(
"src.db.base.get_db_context",
lambda project_id=None: CapturingContext(),
)
disappeared = asyncio.run(
asset_scanner_job._deprecate_missing_assets(
[
{"asset_key": "k8s/pod/prod/current"},
{"asset_key": "prometheus_target/node/192.168.0.110:9100"},
],
("k8s/pod/",),
"00000000-0000-0000-0000-000000000001",
)
)
assert disappeared == 1
assert "last_seen_at <" in str(captured["sql"])
assert captured["parameters"] == {
"patterns": ["k8s/pod/%"],
"seen_keys": ["k8s/pod/prod/current"],
"rid": "00000000-0000-0000-0000-000000000001",
}
def test_runtime_collection_reports_failed_sources_without_reconciling_them(
monkeypatch,
) -> None:
async def fake_kubectl(resource: str, all_namespaces: bool = True):
if resource == "nodes":
raise RuntimeError("nodes unavailable")
return {"items": []}
async def fake_prometheus():
return [], []
async def fake_database_catalog():
return [], []
monkeypatch.setattr(asset_scanner_job, "_fetch_kubectl_json", fake_kubectl)
monkeypatch.setattr(
asset_scanner_job,
"_collect_prometheus_targets",
fake_prometheus,
)
monkeypatch.setattr(
asset_scanner_job,
"_collect_database_catalog_assets",
fake_database_catalog,
)
assets, relationships, prefixes, errors = asyncio.run(
asset_scanner_job._collect_runtime_assets()
)
assert assets == []
assert relationships == []
assert "k8s/node/" not in prefixes
assert "k8s/pod/" in prefixes
assert "prometheus_target/" in prefixes
assert errors == ("k8s_nodes",)
def test_scan_terminal_is_failed_when_any_collector_is_incomplete(monkeypatch) -> None:
terminal: dict[str, object] = {}
async def fake_start(*args, **kwargs):
return "00000000-0000-0000-0000-000000000001"
async def fake_collect():
return [], [], (), ("prometheus_targets",)
async def fake_upsert(*args, **kwargs):
return 0, 0
async def fake_deprecate(*args, **kwargs):
return 0
async def fake_relationships(*args, **kwargs):
return 0
async def fake_coverage(*args, **kwargs):
return None
async def fake_finalize(**kwargs):
terminal.update(kwargs)
monkeypatch.setattr(asset_scanner_job, "_start_discovery_run", fake_start)
monkeypatch.setattr(asset_scanner_job, "_collect_runtime_assets", fake_collect)
monkeypatch.setattr(asset_scanner_job, "_upsert_assets", fake_upsert)
monkeypatch.setattr(
asset_scanner_job,
"_deprecate_missing_assets",
fake_deprecate,
)
monkeypatch.setattr(
asset_scanner_job,
"_upsert_relationships",
fake_relationships,
)
monkeypatch.setattr(
asset_scanner_job,
"_write_coverage_snapshots",
fake_coverage,
)
monkeypatch.setattr(
asset_scanner_job,
"_finalize_discovery_run",
fake_finalize,
)
run_id = asyncio.run(asset_scanner_job.scan_once())
assert run_id == "00000000-0000-0000-0000-000000000001"
assert terminal["status"] == "failed"
assert terminal["error"] == (
"RuntimeError: asset_collector_failures=prometheus_targets"
)
def test_database_catalog_collector_reads_metadata_only(monkeypatch) -> None:
requested_projects: list[str | None] = []

View File

@@ -43,6 +43,13 @@ type Copy = {
audit: string;
aiTrace: string;
sources: string;
latestScan: string;
freshness: string;
runAssets: string;
retired: string;
stale: string;
success: string;
failed: string;
sameRunMissing: string;
unavailable: string;
refresh: string;
@@ -68,6 +75,13 @@ const COPY: Record<"zh" | "en", Copy> = {
audit: "稽核軌跡",
aiTrace: "AI 軌跡",
sources: "資料源",
latestScan: "最新盤點",
freshness: "新鮮度",
runAssets: "本輪資產",
retired: "本輪退役",
stale: "逾期資產",
success: "成功",
failed: "失敗",
sameRunMissing: "尚無同 run 閉環證據",
unavailable: "Live readback 無法取得",
refresh: "重新整理資安控制平面",
@@ -91,6 +105,13 @@ const COPY: Record<"zh" | "en", Copy> = {
audit: "Audit trail",
aiTrace: "AI trace",
sources: "Sources",
latestScan: "Latest scan",
freshness: "Freshness",
runAssets: "Run assets",
retired: "Retired",
stale: "Stale assets",
success: "Success",
failed: "Failed",
sameRunMissing: "No same-run closure evidence",
unavailable: "Live readback unavailable",
refresh: "Refresh security control plane",
@@ -107,6 +128,12 @@ function stageCount(stage: SecurityControlPlaneStage) {
return stage.evidence_count_24h ?? stage.receipt_count_24h ?? 0;
}
function freshnessLabel(ageSeconds: number | null) {
if (ageSeconds === null) return "--";
if (ageSeconds < 60) return `${Math.max(0, Math.round(ageSeconds))}s`;
return `${Math.round(ageSeconds / 60)}m`;
}
async function fetchControlPlane(): Promise<SecurityAssetControlPlanePayload | null> {
const base = getRuntimeApiBaseUrl();
const path = "/api/v1/iwooos/security-asset-control-plane";
@@ -146,6 +173,49 @@ function MetricCell({
);
}
function TruthCell({
label,
value,
icon: Icon,
tone = "neutral",
}: {
label: string;
value: string | number;
icon: typeof ShieldCheck;
tone?: "neutral" | "healthy" | "critical";
}) {
return (
<div className="flex min-w-0 items-center gap-2 bg-white px-3 py-2.5 last:col-span-2 sm:last:col-span-1">
<Icon
className={cn(
"h-3.5 w-3.5 shrink-0",
tone === "healthy"
? "text-[#2f8a50]"
: tone === "critical"
? "text-[#c9584d]"
: "text-[#77736a]"
)}
aria-hidden="true"
/>
<div className="min-w-0">
<div className="truncate text-[9px] font-medium text-[#77736a]">{label}</div>
<div
className={cn(
"mt-0.5 truncate font-mono text-xs font-semibold",
tone === "healthy"
? "text-[#17602a]"
: tone === "critical"
? "text-[#9f2f25]"
: "text-[#141413]"
)}
>
{value}
</div>
</div>
</div>
);
}
function StageStrip({
title,
stages,
@@ -286,6 +356,47 @@ export function SecurityAssetControlPlaneCockpit({
/>
</div>
<div
className="grid grid-cols-2 gap-px border-t border-[#d8d3c7] bg-[#d8d3c7] sm:grid-cols-5"
data-testid="security-asset-discovery-truth"
>
<TruthCell
label={copy.latestScan}
value={
payload?.discovery.latest_status === "success"
? copy.success
: payload
? copy.failed
: "--"
}
icon={Radar}
tone={payload?.discovery.latest_status === "success" ? "healthy" : "critical"}
/>
<TruthCell
label={copy.freshness}
value={payload ? freshnessLabel(payload.discovery.age_seconds) : "--"}
icon={RefreshCw}
tone={payload?.discovery.fresh ? "healthy" : "critical"}
/>
<TruthCell
label={copy.runAssets}
value={payload?.discovery.latest_total_assets ?? "--"}
icon={Database}
/>
<TruthCell
label={copy.retired}
value={payload?.discovery.disappeared_assets ?? "--"}
icon={Boxes}
tone={(payload?.discovery.disappeared_assets ?? 0) > 0 ? "healthy" : "neutral"}
/>
<TruthCell
label={copy.stale}
value={payload?.summary.stale_asset_count ?? "--"}
icon={TriangleAlert}
tone={(payload?.summary.stale_asset_count ?? 0) > 0 ? "critical" : "healthy"}
/>
</div>
{payload ? (
<div className="grid gap-4 p-4 xl:grid-cols-[1.1fr_1.4fr]">
<div className="min-w-0 space-y-4">

View File

@@ -46,10 +46,17 @@ function payload(): SecurityAssetControlPlanePayload {
},
discovery: {
latest_status: "success",
latest_scan_depth: "shallow",
latest_started_at: "2026-07-11T03:49:00+00:00",
latest_ended_at: "2026-07-11T03:50:00+00:00",
latest_success_at: "2026-07-11T03:50:00+00:00",
age_seconds: 600,
freshness_slo_seconds: 7200,
fresh: true,
latest_total_assets: 24,
new_assets: 2,
modified_assets: 22,
disappeared_assets: 1,
},
asset_scopes: [],
nist_csf_functions: [],

View File

@@ -77,10 +77,17 @@ export type SecurityAssetControlPlanePayload = {
summary: SecurityControlPlaneSummary;
discovery: {
latest_status: string;
latest_scan_depth: string;
latest_started_at: string | null;
latest_ended_at: string | null;
latest_success_at: string | null;
age_seconds: number | null;
freshness_slo_seconds: number;
fresh: boolean;
latest_total_assets: number;
new_assets: number;
modified_assets: number;
disappeared_assets: number;
};
asset_scopes: SecurityControlPlaneScope[];
nist_csf_functions: SecurityControlPlaneFunction[];