diff --git a/apps/api/src/jobs/asset_scanner_job.py b/apps/api/src/jobs/asset_scanner_job.py index 16aaa77c3..b0eb1626c 100644 --- a/apps/api/src/jobs/asset_scanner_job.py +++ b/apps/api/src/jobs/asset_scanner_job.py @@ -210,6 +210,7 @@ async def scan_once( "database_catalog", "ai_catalog", "model_registry", + "gitea_bundle_backup", ), scan_depth: str = "shallow", ) -> str: @@ -961,6 +962,19 @@ async def _collect_runtime_assets() -> ( logger.warning("collect_ai_catalog_failed", error=str(e)) collector_errors.append("ai_catalog") + # 12. Gitea repository and backup targets from live Prometheus receipts. + try: + gitea_assets, gitea_relationships = await _collect_gitea_bundle_assets() + assets.extend(gitea_assets) + relationships.extend(gitea_relationships) + if gitea_assets: + reconciled_prefixes.update( + ("gitea/repo/", "backup/gitea-bundle/") + ) + except Exception as e: + logger.warning("collect_gitea_bundle_failed", error=str(e)) + collector_errors.append("gitea_bundle_backup") + return ( list({asset["asset_key"]: asset for asset in assets}.values()), relationships, @@ -969,6 +983,127 @@ async def _collect_runtime_assets() -> ( ) +def _build_gitea_bundle_assets( + payload: dict[str, Any], +) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: + """Project live bundle metrics into repo and backup asset identities.""" + if payload.get("readback_present") is not True: + raise RuntimeError("gitea_bundle_readback_unavailable") + + host = str(payload.get("host") or "").strip() + if not re.fullmatch(r"[A-Za-z0-9_.-]+", host): + raise RuntimeError("gitea_bundle_host_invalid") + + repo_rows = payload.get("repos") or [] + if not isinstance(repo_rows, list) or not repo_rows: + raise RuntimeError("gitea_bundle_repo_rows_missing") + + backup_key = f"backup/gitea-bundle/{host}" + summary = payload.get("summary") or {} + assets: list[dict[str, Any]] = [ + { + "asset_key": backup_key, + "asset_type": "backup_target", + "host": host, + "namespace": None, + "name": f"gitea-bundle-{host}", + "metadata": { + "kind": "GiteaRepositoryBundleTarget", + "status": str(payload.get("status") or "unknown"), + "ready": payload.get("ready") is True, + "expected_repo_count": int(summary.get("expected_repo_count") or 0), + "repo_row_count": int(summary.get("repo_row_count") or len(repo_rows)), + "missing_repo_count": int( + summary.get("expected_repo_missing_count") or 0 + ), + "failed_repo_count": int(summary.get("failed_repo_count") or 0), + "checksum_missing_count": int( + summary.get("checksum_missing_count") or 0 + ), + "bundle_fresh": summary.get("bundle_fresh") is True, + "sample_restore_dry_run_ok": ( + summary.get("sample_restore_dry_run_ok") is True + ), + "source_schema_version": str( + payload.get("schema_version") or "unknown" + ), + "secret_value_read_by_scanner": False, + "repository_content_read_by_scanner": False, + }, + "tags": [ + "source:gitea_bundle_metrics", + "owner:platform_security", + ], + } + ] + relationships: list[dict[str, str]] = [] + + seen_repos: set[str] = set() + for row in repo_rows: + if not isinstance(row, dict): + continue + repo = str(row.get("repo") or "").strip().strip("/") + if not re.fullmatch( + r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + repo, + ): + continue + if repo in seen_repos: + continue + seen_repos.add(repo) + + repo_key = f"gitea/repo/{repo}" + assets.append( + { + "asset_key": repo_key, + "asset_type": "gitea_repo", + "host": host, + "namespace": repo.split("/", 1)[0], + "name": repo, + "metadata": { + "kind": "GiteaRepository", + "bundle_present": row.get("present") is True, + "bundle_verified": row.get("ok") is True, + "head_count": int(row.get("head_count") or 0), + "checksum_digest_present": ( + row.get("checksum_digest_present") is True + ), + "source_schema_version": str( + payload.get("schema_version") or "unknown" + ), + "repository_content_read_by_scanner": False, + "secret_value_read_by_scanner": False, + }, + "tags": [ + "source:gitea_bundle_metrics", + "owner:platform_security", + ], + } + ) + relationships.append( + { + "from_key": repo_key, + "to_key": backup_key, + "relationship_type": "backs_up_to", + } + ) + + if not seen_repos: + raise RuntimeError("gitea_bundle_repo_identities_invalid") + return assets, relationships + + +async def _collect_gitea_bundle_assets() -> ( + tuple[list[dict[str, Any]], list[dict[str, str]]] +): + from src.services.gitea_repo_bundle_backup_readback import ( + load_latest_gitea_repo_bundle_backup_readback, + ) + + payload = await asyncio.to_thread(load_latest_gitea_repo_bundle_backup_readback) + return _build_gitea_bundle_assets(payload) + + def _is_valid_ipv4(s: str) -> bool: """嚴格 IPv4 判斷: 4 段 + 每段 0-255 整數. diff --git a/apps/api/tests/test_asset_scanner_job.py b/apps/api/tests/test_asset_scanner_job.py index b122a743a..87783250d 100644 --- a/apps/api/tests/test_asset_scanner_job.py +++ b/apps/api/tests/test_asset_scanner_job.py @@ -374,6 +374,9 @@ def test_runtime_collection_reports_failed_sources_without_reconciling_them( async def fake_ai_catalog(): return [], [] + async def fake_gitea_bundle(): + return [], [] + monkeypatch.setattr(asset_scanner_job, "_fetch_kubectl_json", fake_kubectl) monkeypatch.setattr( asset_scanner_job, @@ -390,6 +393,11 @@ def test_runtime_collection_reports_failed_sources_without_reconciling_them( "_collect_ai_catalog_assets", fake_ai_catalog, ) + monkeypatch.setattr( + asset_scanner_job, + "_collect_gitea_bundle_assets", + fake_gitea_bundle, + ) assets, relationships, prefixes, errors = asyncio.run( asset_scanner_job._collect_runtime_assets() @@ -636,6 +644,11 @@ def test_runtime_collection_integrates_new_asset_types_and_reconciliation_prefix "_collect_ai_catalog_assets", empty_collector, ) + monkeypatch.setattr( + asset_scanner_job, + "_collect_gitea_bundle_assets", + empty_collector, + ) assets, relationships, prefixes, errors = asyncio.run( asset_scanner_job._collect_runtime_assets() @@ -769,6 +782,71 @@ def test_ai_catalog_collector_reads_aggregates_and_model_names_only( ] +def test_gitea_bundle_readback_builds_repo_and_backup_assets() -> None: + assets, relationships = asset_scanner_job._build_gitea_bundle_assets( + { + "schema_version": "gitea_repo_bundle_backup_readback_v1", + "status": "gitea_repo_bundle_backup_readback_ready", + "host": "188", + "readback_present": True, + "ready": True, + "summary": { + "expected_repo_count": 2, + "repo_row_count": 2, + "expected_repo_missing_count": 0, + "failed_repo_count": 0, + "checksum_missing_count": 0, + "bundle_fresh": True, + "sample_restore_dry_run_ok": True, + }, + "repos": [ + { + "repo": "wooo/awoooi", + "present": True, + "ok": True, + "head_count": 196, + "checksum_digest_present": True, + }, + { + "repo": "wooo/tsenyang-website", + "present": True, + "ok": True, + "head_count": 2, + "checksum_digest_present": True, + }, + ], + } + ) + + by_type: dict[str, list[dict]] = {} + for asset in assets: + by_type.setdefault(asset["asset_type"], []).append(asset) + + assert len(by_type["gitea_repo"]) == 2 + assert len(by_type["backup_target"]) == 1 + assert by_type["backup_target"][0]["metadata"]["ready"] is True + assert all( + asset["metadata"]["repository_content_read_by_scanner"] is False + for asset in assets + ) + assert all( + asset_scanner_job._resolve_owner_team(asset)[0] == "platform_security" + for asset in assets + ) + assert relationships == [ + { + "from_key": "gitea/repo/wooo/awoooi", + "to_key": "backup/gitea-bundle/188", + "relationship_type": "backs_up_to", + }, + { + "from_key": "gitea/repo/wooo/tsenyang-website", + "to_key": "backup/gitea-bundle/188", + "relationship_type": "backs_up_to", + }, + ] + + def test_scan_terminal_is_failed_when_any_collector_is_incomplete(monkeypatch) -> None: terminal: dict[str, object] = {}