fix(iwooos): isolate control plane source reads
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m34s
CD Pipeline / build-and-deploy (push) Failing after 8m21s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 15s
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
ogt
2026-07-11 19:59:18 +08:00
parent 5e286f25be
commit ea85739951
2 changed files with 191 additions and 37 deletions

View File

@@ -1069,6 +1069,9 @@ def build_unavailable_iwooos_security_asset_control_plane(
"source_status": "live_database_unavailable",
"reason_code": reason_code,
"summary": {
"source_count": 9,
"ready_source_count": 0,
"unavailable_source_count": 9,
"managed_asset_count": 0,
"expected_asset_type_count": len(_ASSET_TYPES),
"present_asset_type_count": 0,
@@ -1212,6 +1215,24 @@ def build_unavailable_iwooos_security_asset_control_plane(
"accepted_ai_trace_count_24h": 0,
"immutable_external_audit_store_evidenced": False,
},
"source_health": [
{
"source_id": source_id,
"status": "unavailable",
"reason_code": reason_code,
}
for source_id in (
"asset_discovery",
"asset_inventory",
"automation_coverage",
"asset_relationship",
"security_compliance",
"automation_operations",
"ai_decision_trace",
"siem_runtime",
"audit_runtime",
)
],
"work_items": [
_work_item(
"AIA-P0-006-01",
@@ -1250,6 +1271,44 @@ def build_unavailable_iwooos_security_asset_control_plane(
}
async def _read_public_source(
db: Any,
*,
source_id: str,
statement: Any,
result_mode: str,
default: Any,
) -> tuple[Any, dict[str, Any]]:
"""Isolate one aggregate query so an optional source cannot abort the readback."""
try:
async with db.begin_nested():
result = await db.execute(statement)
if result_mode == "all":
value = result.fetchall()
elif result_mode == "one":
value = result.one()
elif result_mode == "one_or_none":
value = result.one_or_none()
else:
raise ValueError("unsupported_result_mode")
return value, {
"source_id": source_id,
"status": "ready",
"reason_code": None,
}
except Exception as exc: # noqa: BLE001 - isolate and redact source failures
logger.warning(
"iwooos_security_asset_control_plane_source_unavailable",
source_id=source_id,
error_type=type(exc).__name__,
)
return default, {
"source_id": source_id,
"status": "unavailable",
"reason_code": f"{source_id}_query_failed",
}
class IwoooSSecurityAssetControlPlaneService:
"""Read aggregate production security state with a bounded DB timeout."""
@@ -1274,8 +1333,12 @@ class IwoooSSecurityAssetControlPlaneService:
async def _load_snapshot(self) -> dict[str, Any]:
async with get_db_context("awoooi") as db:
discovery_result = await db.execute(
_sql(
source_health: list[dict[str, Any]] = []
discovery_row, health = await _read_public_source(
db,
source_id="asset_discovery",
statement=_sql(
"""
SELECT
latest.status AS latest_status,
@@ -1303,12 +1366,16 @@ class IwoooSSecurityAssetControlPlaneService:
LIMIT 1
) successful ON true
"""
)
),
result_mode="one_or_none",
default=None,
)
discovery_row = discovery_result.one_or_none()
source_health.append(health)
inventory_result = await db.execute(
_sql(
inventory_rows, health = await _read_public_source(
db,
source_id="asset_inventory",
statement=_sql(
"""
SELECT
asset_type,
@@ -1324,12 +1391,16 @@ class IwoooSSecurityAssetControlPlaneService:
GROUP BY asset_type
ORDER BY asset_type
"""
)
),
result_mode="all",
default=[],
)
inventory_rows = inventory_result.fetchall()
source_health.append(health)
coverage_result = await db.execute(
_sql(
coverage_rows, health = await _read_public_source(
db,
source_id="automation_coverage",
statement=_sql(
"""
SELECT dimension, coverage_status AS status, count(*) AS cnt
FROM asset_coverage_snapshot
@@ -1343,12 +1414,16 @@ class IwoooSSecurityAssetControlPlaneService:
GROUP BY dimension, coverage_status
ORDER BY dimension, coverage_status
"""
)
),
result_mode="all",
default=[],
)
coverage_rows = coverage_result.fetchall()
source_health.append(health)
relationship_result = await db.execute(
_sql(
relationship_row, health = await _read_public_source(
db,
source_id="asset_relationship",
statement=_sql(
"""
SELECT
(SELECT count(*) FROM asset_relationship WHERE is_active) AS relationship_count,
@@ -1363,12 +1438,16 @@ class IwoooSSecurityAssetControlPlaneService:
FROM asset_inventory asset
WHERE asset.lifecycle_state = 'active'
"""
)
),
result_mode="one",
default={},
)
relationship_row = relationship_result.one()
source_health.append(health)
compliance_result = await db.execute(
_sql(
compliance_rows, health = await _read_public_source(
db,
source_id="security_compliance",
statement=_sql(
"""
WITH latest AS (
SELECT DISTINCT ON (asset_id, dimension)
@@ -1381,12 +1460,16 @@ class IwoooSSecurityAssetControlPlaneService:
GROUP BY dimension, status
ORDER BY dimension, status
"""
)
),
result_mode="all",
default=[],
)
compliance_rows = compliance_result.fetchall()
source_health.append(health)
automation_result = await db.execute(
_sql(
automation_rows, health = await _read_public_source(
db,
source_id="automation_operations",
statement=_sql(
"""
SELECT operation_type, status, count(*) AS cnt
FROM automation_operation_log
@@ -1394,12 +1477,16 @@ class IwoooSSecurityAssetControlPlaneService:
GROUP BY operation_type, status
ORDER BY operation_type, status
"""
)
),
result_mode="all",
default=[],
)
automation_rows = automation_result.fetchall()
source_health.append(health)
ai_trace_result = await db.execute(
_sql(
ai_trace_row, health = await _read_public_source(
db,
source_id="ai_decision_trace",
statement=_sql(
"""
SELECT
count(*) AS trace_count,
@@ -1408,12 +1495,16 @@ class IwoooSSecurityAssetControlPlaneService:
JOIN automation_operation_log operation ON operation.op_id = trace.op_id
WHERE operation.created_at >= NOW() - INTERVAL '24 hours'
"""
)
),
result_mode="one",
default={},
)
ai_trace_row = ai_trace_result.one()
source_health.append(health)
siem_result = await db.execute(
_sql(
siem_row, health = await _read_public_source(
db,
source_id="siem_runtime",
statement=_sql(
"""
SELECT
(SELECT count(*) FROM alert_rule_catalog) AS rule_total,
@@ -1454,12 +1545,16 @@ class IwoooSSecurityAssetControlPlaneService:
AND resolved_at >= NOW() - INTERVAL '24 hours')
AS mttr_minutes_24h
"""
)
),
result_mode="one",
default={},
)
siem_row = siem_result.one()
source_health.append(health)
audit_result = await db.execute(
_sql(
audit_row, health = await _read_public_source(
db,
source_id="audit_runtime",
statement=_sql(
"""
SELECT
(SELECT count(*) FROM audit_logs
@@ -1486,9 +1581,11 @@ class IwoooSSecurityAssetControlPlaneService:
AND ai_analysis IS NOT NULL)
AS asset_change_ai_analysis_count_24h
"""
)
),
result_mode="one",
default={},
)
audit_row = audit_result.one()
source_health.append(health)
return build_iwooos_security_asset_control_plane(
discovery_row=discovery_row,
@@ -1500,6 +1597,7 @@ class IwoooSSecurityAssetControlPlaneService:
ai_trace_row=ai_trace_row,
siem_row=siem_row,
audit_row=audit_row,
source_health=source_health,
)