From 248cf07ce11feda858671971c6bd255aff3beb68 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 14:58:12 +0800 Subject: [PATCH] feat(security): inventory public gateway certificates --- apps/api/src/jobs/asset_scanner_job.py | 188 ++++++++++++++++++ .../iwooos_security_asset_control_plane.py | 75 ++++++- apps/api/tests/test_asset_scanner_job.py | 110 ++++++++++ ...est_iwooos_security_asset_control_plane.py | 56 ++++++ 4 files changed, 427 insertions(+), 2 deletions(-) diff --git a/apps/api/src/jobs/asset_scanner_job.py b/apps/api/src/jobs/asset_scanner_job.py index ca3f3f452..5e3b63fda 100644 --- a/apps/api/src/jobs/asset_scanner_job.py +++ b/apps/api/src/jobs/asset_scanner_job.py @@ -212,6 +212,7 @@ async def scan_once( "model_registry", "gitea_bundle_backup", "gitea_ci", + "domain_tls_inventory", ), scan_depth: str = "shallow", ) -> str: @@ -1032,6 +1033,180 @@ def _collect_ingress_assets( return list(assets_by_key.values()), relationships +def _normalize_public_domain(value: Any) -> str | None: + domain = str(value or "").strip().lower().rstrip(".") + if not domain or len(domain) > 253 or any(ord(char) < 33 for char in domain): + return None + labels = domain.split(".") + if len(labels) < 2: + return None + if any( + not label + or len(label) > 63 + or label.startswith("-") + or label.endswith("-") + or re.fullmatch(r"[a-z0-9-]+", label) is None + for label in labels + ): + return None + return domain + + +def _build_domain_tls_inventory_assets( + payload: dict[str, Any], +) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: + """Project repo-known gateway TLS identities without retaining raw paths.""" + if payload.get("schema_version") != "domain_tls_certbot_inventory_v1": + raise RuntimeError("domain_tls_inventory_schema_invalid") + if payload.get("mode") != "repo_only_from_nginx_source_of_truth": + raise RuntimeError("domain_tls_inventory_source_mode_invalid") + boundaries = payload.get("execution_boundaries") or {} + for boundary in ( + "secret_value_collected", + "live_tls_probe_executed", + "certbot_renew_executed", + "nginx_reload_executed", + "host_write_executed", + ): + if boundaries.get(boundary) is not False: + raise RuntimeError(f"domain_tls_inventory_boundary_invalid:{boundary}") + + domain_rows = payload.get("managed_domains") or [] + if not isinstance(domain_rows, list) or not domain_rows: + raise RuntimeError("domain_tls_inventory_domains_missing") + + source_generated_at = str(payload.get("generated_at") or "unknown") + domains: dict[str, dict[str, Any]] = {} + certificates: dict[str, dict[str, Any]] = {} + relationship_keys: set[tuple[str, str, str]] = set() + + for row in domain_rows: + if not isinstance(row, dict): + raise RuntimeError("domain_tls_inventory_domain_row_invalid") + domain = _normalize_public_domain(row.get("domain")) + if domain is None or domain in domains: + raise RuntimeError("domain_tls_inventory_domain_identity_invalid") + + domain_key = f"repo/nginx-domain/{domain}" + domains[domain] = { + "asset_key": domain_key, + "asset_type": "website", + "host": domain, + "namespace": None, + "name": domain, + "metadata": { + "kind": "RepoKnownPublicGatewayDomain", + "source_schema_version": payload["schema_version"], + "source_generated_at": source_generated_at, + "control_tier": str(row.get("control_tier") or "unknown"), + "tls_certificate_path_present": ( + row.get("tls_certificate_path_present") is True + ), + "certificate_owner_confirmation_required": ( + row.get("certificate_owner_confirmation_required") is True + ), + "acme_challenge_present": row.get("acme_challenge_present") is True, + "live_tls_probe_status": str( + row.get("live_tls_probe_status") or "not_executed" + ), + "dns_resolution_status": str( + row.get("dns_resolution_status") or "not_executed" + ), + "certbot_renewal_status": str( + row.get("certbot_renewal_status") or "not_executed" + ), + "owner_review_status": str( + row.get("owner_review_status") or "unknown" + ), + "nginx_config_content_read_by_scanner": False, + "request_or_response_body_read": False, + "secret_value_read": False, + }, + "tags": [ + "source:domain_tls_certbot_inventory", + "layer:public_gateway", + ], + } + + raw_paths = row.get("certificate_paths") or [] + if not isinstance(raw_paths, list): + raise RuntimeError("domain_tls_inventory_certificate_paths_invalid") + for raw_path in raw_paths: + certificate_path = str(raw_path or "") + if ( + not certificate_path + or len(certificate_path) > 2048 + or any(ord(char) < 32 for char in certificate_path) + ): + raise RuntimeError("domain_tls_inventory_certificate_path_invalid") + fingerprint = hashlib.sha256(certificate_path.encode("utf-8")).hexdigest() + certificate_key = f"repo/tls-certificate/{fingerprint[:24]}" + certificates.setdefault( + certificate_key, + { + "asset_key": certificate_key, + "asset_type": "certificate", + "host": None, + "namespace": None, + "name": f"managed-certificate-{fingerprint[:12]}", + "metadata": { + "kind": "RepoKnownCertificateReference", + "source_schema_version": payload["schema_version"], + "source_generated_at": source_generated_at, + "certificate_path_fingerprint": fingerprint, + "live_tls_probe_evidenced": False, + "certificate_material_read": False, + "private_key_material_read": False, + "raw_certificate_path_stored": False, + "secret_value_read": False, + }, + "tags": [ + "source:domain_tls_certbot_inventory", + "material:not_read", + ], + }, + ) + relationship_keys.add((domain_key, certificate_key, "authenticates_via")) + + summary = payload.get("summary") or {} + if int(summary.get("managed_domain_count") or 0) != len(domains): + raise RuntimeError("domain_tls_inventory_domain_count_mismatch") + if int(summary.get("unique_certificate_path_count") or 0) != len(certificates): + raise RuntimeError("domain_tls_inventory_certificate_count_mismatch") + + relationships = [ + { + "from_key": from_key, + "to_key": to_key, + "relationship_type": relationship_type, + } + for from_key, to_key, relationship_type in sorted(relationship_keys) + ] + return [*domains.values(), *certificates.values()], relationships + + +def _load_domain_tls_inventory_payload(path: Path | None = None) -> dict[str, Any]: + if path is None: + relative = Path("docs/security/domain-tls-certbot-inventory.snapshot.json") + candidates = (Path.cwd() / relative, Path(__file__).resolve().parents[4] / relative) + path = next((candidate for candidate in candidates if candidate.is_file()), None) + if path is None or not path.is_file(): + raise RuntimeError("domain_tls_inventory_snapshot_missing") + if path.stat().st_size > 5_000_000: + raise RuntimeError("domain_tls_inventory_snapshot_too_large") + payload = _json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise RuntimeError("domain_tls_inventory_payload_invalid") + return payload + + +async def _collect_domain_tls_inventory_assets() -> ( + tuple[list[dict[str, Any]], list[dict[str, str]]] +): + payload = await asyncio.to_thread(_load_domain_tls_inventory_payload) + return _build_domain_tls_inventory_assets(payload) + + async def _collect_runtime_assets() -> ( tuple[ list[dict[str, Any]], @@ -1374,6 +1549,19 @@ async def _collect_runtime_assets() -> ( logger.warning("collect_gitea_ci_failed", error=str(e)) collector_errors.append("gitea_ci") + # 14. Public gateway domain/TLS identities from committed structured inventory. + try: + tls_assets, tls_relationships = await _collect_domain_tls_inventory_assets() + assets.extend(tls_assets) + relationships.extend(tls_relationships) + if tls_assets: + reconciled_prefixes.update( + ("repo/nginx-domain/", "repo/tls-certificate/") + ) + except Exception as e: + logger.warning("collect_domain_tls_inventory_failed", error=str(e)) + collector_errors.append("domain_tls_inventory") + return ( list({asset["asset_key"]: asset for asset in assets}.values()), relationships, diff --git a/apps/api/src/services/iwooos_security_asset_control_plane.py b/apps/api/src/services/iwooos_security_asset_control_plane.py index 7174f9a84..297bd96de 100644 --- a/apps/api/src/services/iwooos_security_asset_control_plane.py +++ b/apps/api/src/services/iwooos_security_asset_control_plane.py @@ -368,6 +368,8 @@ def _build_security_program_domains( github_supply_chain_compliant_percent: int, forbidden_github_supply_chain_asset_count: int, package_digest_unpinned_count: int, + certificate_live_verified_percent: int, + certificate_live_unverified_count: int, ) -> list[dict[str, Any]]: scope_by_id = {row["scope_id"]: row for row in asset_scopes} function_by_id = {row["function_id"]: row for row in control_functions} @@ -397,6 +399,11 @@ def _build_security_program_domains( supply_chain_gap_code = "runtime_image_digest_pin_missing" else: supply_chain_gap_code = "sbom_signature_dependency_evidence_missing" + network_tls_gap_code = ( + "certificate_live_tls_probe_missing" + if certificate_live_unverified_count > 0 + else "network_dns_tls_runtime_evidence_missing" + ) domain_values = ( ( @@ -453,12 +460,13 @@ def _build_security_program_domains( average( 100 if by_type.get("network", 0) > 0 else 0, 100 if by_type.get("certificate", 0) > 0 else 0, + certificate_live_verified_percent, compliance_percent("ssl_cert_valid"), ), by_type.get("network", 0) + by_type.get("certificate", 0) + compliance_total("ssl_cert_valid"), - "network_dns_tls_runtime_evidence_missing", + network_tls_gap_code, ), ( "cloud_container_k8s", @@ -675,6 +683,10 @@ def build_iwooos_security_asset_control_plane( int(_value(row, "unpinned_package_count", 0) or 0) for row in inventory_source ) + certificate_live_unverified_count = sum( + int(_value(row, "unverified_certificate_count", 0) or 0) + for row in inventory_source + ) present_types = { asset_type for asset_type in _ASSET_TYPES if by_type.get(asset_type, 0) > 0 } @@ -886,6 +898,15 @@ def build_iwooos_security_asset_control_plane( forbidden_github_supply_chain_asset_count ), package_digest_unpinned_count=package_digest_unpinned_count, + certificate_live_verified_percent=_percent( + max( + 0, + by_type.get("certificate", 0) + - certificate_live_unverified_count, + ), + by_type.get("certificate", 0), + ), + certificate_live_unverified_count=certificate_live_unverified_count, ) work_items: list[dict[str, Any]] = [] @@ -941,6 +962,20 @@ def build_iwooos_security_asset_control_plane( ), ) ) + if certificate_live_unverified_count > 0: + work_items.append( + _work_item( + "AIA-P0-006-02B", + "P0", + "certificate_live_verification", + "驗證 public TLS 憑證", + certificate_live_unverified_count, + ( + "執行 bounded TLS handshake metadata probe，驗證 hostname、" + "chain 與 expiry；不得讀取 private key 或 certificate volume。" + ), + ) + ) if coverage_total == 0 or coverage_unknown > 0: work_items.append( _work_item( @@ -1094,6 +1129,9 @@ def build_iwooos_security_asset_control_plane( "forbidden_github_supply_chain_asset_count": ( forbidden_github_supply_chain_asset_count ), + "certificate_live_unverified_count": ( + certificate_live_unverified_count + ), "automation_coverage_green_percent": _percent( coverage_green, coverage_total ), @@ -1176,6 +1214,22 @@ def build_iwooos_security_asset_control_plane( "registry_api_called": False, "github_api_used": False, }, + "certificate_inventory": { + "certificate_asset_count": by_type.get("certificate", 0), + "live_tls_verified_count": max( + 0, + by_type.get("certificate", 0) + - certificate_live_unverified_count, + ), + "live_tls_unverified_count": certificate_live_unverified_count, + "live_tls_health_proven": ( + by_type.get("certificate", 0) > 0 + and certificate_live_unverified_count == 0 + ), + "raw_certificate_path_returned": False, + "certificate_material_read": False, + "private_key_material_read": False, + }, "source_health": source_health_rows, "work_items": work_items, "boundaries": { @@ -1217,6 +1271,7 @@ def build_unavailable_iwooos_security_asset_control_plane( "relationship_count": 0, "package_digest_unpinned_count": 0, "forbidden_github_supply_chain_asset_count": 0, + "certificate_live_unverified_count": 0, "automation_coverage_green_percent": 0, "automation_coverage_unknown_count": 0, "compliance_violation_count": 0, @@ -1380,6 +1435,15 @@ def build_unavailable_iwooos_security_asset_control_plane( "registry_api_called": False, "github_api_used": False, }, + "certificate_inventory": { + "certificate_asset_count": 0, + "live_tls_verified_count": 0, + "live_tls_unverified_count": 0, + "live_tls_health_proven": False, + "raw_certificate_path_returned": False, + "certificate_material_read": False, + "private_key_material_read": False, + }, "source_health": [ { "source_id": source_id, @@ -1601,7 +1665,14 @@ class IwoooSSecurityAssetControlPlaneService: metadata->>'digest_pinned', 'false' ) <> 'true' - ) AS unpinned_package_count + ) AS unpinned_package_count, + count(*) FILTER ( + WHERE asset_type = 'certificate' + AND COALESCE( + metadata->>'live_tls_probe_evidenced', + 'false' + ) <> 'true' + ) AS unverified_certificate_count FROM asset_inventory WHERE lifecycle_state = 'active' GROUP BY asset_type diff --git a/apps/api/tests/test_asset_scanner_job.py b/apps/api/tests/test_asset_scanner_job.py index a7026209f..2aa5ffa7a 100644 --- a/apps/api/tests/test_asset_scanner_job.py +++ b/apps/api/tests/test_asset_scanner_job.py @@ -380,6 +380,9 @@ def test_runtime_collection_reports_failed_sources_without_reconciling_them( async def fake_gitea_ci(): return [], [] + async def fake_domain_tls(): + return [], [] + monkeypatch.setattr(asset_scanner_job, "_fetch_kubectl_json", fake_kubectl) monkeypatch.setattr( asset_scanner_job, @@ -406,6 +409,11 @@ def test_runtime_collection_reports_failed_sources_without_reconciling_them( "_collect_gitea_ci_assets", fake_gitea_ci, ) + monkeypatch.setattr( + asset_scanner_job, + "_collect_domain_tls_inventory_assets", + fake_domain_tls, + ) assets, relationships, prefixes, errors = asyncio.run( asset_scanner_job._collect_runtime_assets() @@ -528,6 +536,103 @@ def test_ingress_route_identity_is_stable_when_manifest_order_changes() -> None: assert len(set(forward_keys.values())) == 2 +def test_domain_tls_inventory_projects_fingerprints_without_raw_paths() -> None: + payload = { + "schema_version": "domain_tls_certbot_inventory_v1", + "generated_at": "2026-07-14T00:00:00+08:00", + "mode": "repo_only_from_nginx_source_of_truth", + "execution_boundaries": { + "secret_value_collected": False, + "live_tls_probe_executed": False, + "certbot_renew_executed": False, + "nginx_reload_executed": False, + "host_write_executed": False, + }, + "summary": { + "managed_domain_count": 2, + "unique_certificate_path_count": 1, + }, + "managed_domains": [ + { + "domain": "app.example.test", + "control_tier": "C0", + "tls_certificate_path_present": True, + "certificate_owner_confirmation_required": False, + "acme_challenge_present": True, + "live_tls_probe_status": "not_executed", + "dns_resolution_status": "not_executed", + "certbot_renewal_status": "not_executed", + "owner_review_status": "repo_only_ready_for_owner_review", + "certificate_paths": [ + "/etc/letsencrypt/live/example/fullchain.pem", + ], + "certificate_key_paths": [ + "/etc/letsencrypt/live/example/privkey.pem", + ], + }, + { + "domain": "api.example.test", + "control_tier": "C0", + "tls_certificate_path_present": True, + "certificate_owner_confirmation_required": True, + "acme_challenge_present": True, + "live_tls_probe_status": "not_executed", + "dns_resolution_status": "not_executed", + "certbot_renewal_status": "not_executed", + "owner_review_status": "repo_only_owner_confirmation_required", + "certificate_paths": [ + "/etc/letsencrypt/live/example/fullchain.pem", + ], + "certificate_key_paths": [ + "/etc/letsencrypt/live/example/privkey.pem", + ], + }, + ], + } + + assets, relationships = asset_scanner_job._build_domain_tls_inventory_assets( + payload + ) + + assert len(assets) == 3 + assert {asset["asset_type"] for asset in assets} == {"website", "certificate"} + certificate = next(asset for asset in assets if asset["asset_type"] == "certificate") + assert certificate["metadata"]["raw_certificate_path_stored"] is False + assert certificate["metadata"]["certificate_material_read"] is False + assert certificate["metadata"]["private_key_material_read"] is False + assert len(relationships) == 2 + + public_text = json.dumps({"assets": assets, "relationships": relationships}) + assert "/etc/letsencrypt" not in public_text + assert "privkey.pem" not in public_text + + +def test_domain_tls_inventory_fails_closed_when_secret_boundary_is_not_false() -> ( + None +): + payload = { + "schema_version": "domain_tls_certbot_inventory_v1", + "mode": "repo_only_from_nginx_source_of_truth", + "execution_boundaries": { + "secret_value_collected": True, + "live_tls_probe_executed": False, + "certbot_renew_executed": False, + "nginx_reload_executed": False, + "host_write_executed": False, + }, + "managed_domains": [{"domain": "app.example.test"}], + } + + try: + asset_scanner_job._build_domain_tls_inventory_assets(payload) + except RuntimeError as exc: + assert str(exc) == ( + "domain_tls_inventory_boundary_invalid:secret_value_collected" + ) + else: + raise AssertionError("domain TLS inventory must fail closed on secret boundary") + + def test_k8s_metadata_assets_never_read_volume_secret_or_command_data() -> None: pvc = asset_scanner_job._build_pvc_asset( { @@ -980,6 +1085,11 @@ services: "_collect_gitea_ci_assets", empty_collector, ) + monkeypatch.setattr( + asset_scanner_job, + "_collect_domain_tls_inventory_assets", + empty_collector, + ) assets, relationships, prefixes, errors = asyncio.run( asset_scanner_job._collect_runtime_assets() diff --git a/apps/api/tests/test_iwooos_security_asset_control_plane.py b/apps/api/tests/test_iwooos_security_asset_control_plane.py index 428405ec4..3eaf6482b 100644 --- a/apps/api/tests/test_iwooos_security_asset_control_plane.py +++ b/apps/api/tests/test_iwooos_security_asset_control_plane.py @@ -170,6 +170,8 @@ def test_builds_live_asset_siem_audit_and_ai_control_plane_without_false_closure assert payload["supply_chain"]["raw_image_identity_returned"] is False assert payload["supply_chain"]["registry_api_called"] is False assert payload["supply_chain"]["github_api_used"] is False + assert payload["certificate_inventory"]["live_tls_health_proven"] is False + assert payload["certificate_inventory"]["private_key_material_read"] is False assert any( item["work_item_id"] == "AIA-P0-006-02" for item in payload["work_items"] ) @@ -296,6 +298,59 @@ def test_unknown_coverage_and_compliance_create_p0_work_items() -> None: } <= work_item_ids +def test_certificate_assets_remain_open_until_live_tls_probe_is_evidenced() -> None: + now = datetime(2026, 7, 11, 4, 0, tzinfo=UTC) + payload = build_iwooos_security_asset_control_plane( + discovery_row=_row( + latest_status="success", + latest_scan_depth="full", + latest_success_ended_at=now - timedelta(minutes=1), + latest_total_assets=10, + ), + inventory_rows=[ + _row( + asset_type="certificate", + cnt=10, + unowned_count=10, + stale_count=0, + unverified_certificate_count=10, + ), + ], + coverage_rows=[], + relationship_row=_row(relationship_count=0, orphan_asset_count=10), + compliance_rows=[], + automation_rows=[], + ai_trace_row=_row(trace_count=0, accepted_trace_count=0), + siem_row=_row(), + audit_row=_row(), + generated_at=now, + ) + + assert payload["summary"]["certificate_live_unverified_count"] == 10 + assert payload["certificate_inventory"] == { + "certificate_asset_count": 10, + "live_tls_verified_count": 0, + "live_tls_unverified_count": 10, + "live_tls_health_proven": False, + "raw_certificate_path_returned": False, + "certificate_material_read": False, + "private_key_material_read": False, + } + work_item = next( + item + for item in payload["work_items"] + if item["work_item_id"] == "AIA-P0-006-02B" + ) + assert work_item["gap_count"] == 10 + domains = { + domain["domain_id"]: domain for domain in payload["security_program_domains"] + } + assert domains["network_dns_tls"]["gap_code"] == ( + "certificate_live_tls_probe_missing" + ) + assert domains["network_dns_tls"]["status"] != "controlled" + + def test_partial_source_failure_is_visible_without_erasing_core_inventory() -> None: source_health = [ {"source_id": "asset_discovery", "status": "ready", "reason_code": None}, @@ -491,6 +546,7 @@ def test_service_bounds_source_concurrency_for_database_budget(monkeypatch) -> N ) assert "forbidden_source_domain_detected" in inventory_query assert "digest_pinned" in inventory_query + assert "live_tls_probe_evidenced" in inventory_query def test_public_api_returns_live_aggregate(monkeypatch) -> None: