wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity

feat(security): ingest gitea backup assets

Browse Source
This commit is contained in:
ogt
2026-07-14 14:16:54 +08:00
parent 69d2e4381d
commit f963452b3e
2 changed files with 213 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
apps/api/src/jobs/asset_scanner_job.py
View File

@@ -210,6 +210,7 @@ async def scan_once(
"database_catalog",
"ai_catalog",
"model_registry",
"gitea_bundle_backup",
),
scan_depth: str = "shallow",
) -> str:
@@ -961,6 +962,19 @@ async def _collect_runtime_assets() -> (
logger.warning("collect_ai_catalog_failed", error=str(e))
collector_errors.append("ai_catalog")
# 12. Gitea repository and backup targets from live Prometheus receipts.
try:
gitea_assets, gitea_relationships = await _collect_gitea_bundle_assets()
assets.extend(gitea_assets)
relationships.extend(gitea_relationships)
if gitea_assets:
reconciled_prefixes.update(
("gitea/repo/", "backup/gitea-bundle/")
)
except Exception as e:
logger.warning("collect_gitea_bundle_failed", error=str(e))
collector_errors.append("gitea_bundle_backup")
return (
list({asset["asset_key"]: asset for asset in assets}.values()),
relationships,
@@ -969,6 +983,127 @@ async def _collect_runtime_assets() -> (
)
def _build_gitea_bundle_assets(
payload: dict[str, Any],
) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:
"""Project live bundle metrics into repo and backup asset identities."""
if payload.get("readback_present") is not True:
raise RuntimeError("gitea_bundle_readback_unavailable")
host = str(payload.get("host") or "").strip()
if not re.fullmatch(r"[A-Za-z0-9_.-]+", host):
raise RuntimeError("gitea_bundle_host_invalid")
repo_rows = payload.get("repos") or []
if not isinstance(repo_rows, list) or not repo_rows:
raise RuntimeError("gitea_bundle_repo_rows_missing")
backup_key = f"backup/gitea-bundle/{host}"
summary = payload.get("summary") or {}
assets: list[dict[str, Any]] = [
{
"asset_key": backup_key,
"asset_type": "backup_target",
"host": host,
"namespace": None,
"name": f"gitea-bundle-{host}",
"metadata": {
"kind": "GiteaRepositoryBundleTarget",
"status": str(payload.get("status") or "unknown"),
"ready": payload.get("ready") is True,
"expected_repo_count": int(summary.get("expected_repo_count") or 0),
"repo_row_count": int(summary.get("repo_row_count") or len(repo_rows)),
"missing_repo_count": int(
summary.get("expected_repo_missing_count") or 0
),
"failed_repo_count": int(summary.get("failed_repo_count") or 0),
"checksum_missing_count": int(
summary.get("checksum_missing_count") or 0
),
"bundle_fresh": summary.get("bundle_fresh") is True,
"sample_restore_dry_run_ok": (
summary.get("sample_restore_dry_run_ok") is True
),
"source_schema_version": str(
payload.get("schema_version") or "unknown"
),
"secret_value_read_by_scanner": False,
"repository_content_read_by_scanner": False,
},
"tags": [
"source:gitea_bundle_metrics",
"owner:platform_security",
],
}
]
relationships: list[dict[str, str]] = []
seen_repos: set[str] = set()
for row in repo_rows:
if not isinstance(row, dict):
continue
repo = str(row.get("repo") or "").strip().strip("/")
if not re.fullmatch(
r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+",
repo,
):
continue
if repo in seen_repos:
continue
seen_repos.add(repo)
repo_key = f"gitea/repo/{repo}"
assets.append(
{
"asset_key": repo_key,
"asset_type": "gitea_repo",
"host": host,
"namespace": repo.split("/", 1)[0],
"name": repo,
"metadata": {
"kind": "GiteaRepository",
"bundle_present": row.get("present") is True,
"bundle_verified": row.get("ok") is True,
"head_count": int(row.get("head_count") or 0),
"checksum_digest_present": (
row.get("checksum_digest_present") is True
),
"source_schema_version": str(
payload.get("schema_version") or "unknown"
),
"repository_content_read_by_scanner": False,
"secret_value_read_by_scanner": False,
},
"tags": [
"source:gitea_bundle_metrics",
"owner:platform_security",
],
}
)
relationships.append(
{
"from_key": repo_key,
"to_key": backup_key,
"relationship_type": "backs_up_to",
}
)
if not seen_repos:
raise RuntimeError("gitea_bundle_repo_identities_invalid")
return assets, relationships
async def _collect_gitea_bundle_assets() -> (
tuple[list[dict[str, Any]], list[dict[str, str]]]
):
from src.services.gitea_repo_bundle_backup_readback import (
load_latest_gitea_repo_bundle_backup_readback,
)
payload = await asyncio.to_thread(load_latest_gitea_repo_bundle_backup_readback)
return _build_gitea_bundle_assets(payload)
def _is_valid_ipv4(s: str) -> bool:
"""嚴格 IPv4 判斷: 4 段 + 每段 0-255 整數.

78
apps/api/tests/test_asset_scanner_job.py
View File

@@ -374,6 +374,9 @@ def test_runtime_collection_reports_failed_sources_without_reconciling_them(
async def fake_ai_catalog():
return [], []
async def fake_gitea_bundle():
return [], []
monkeypatch.setattr(asset_scanner_job, "_fetch_kubectl_json", fake_kubectl)
monkeypatch.setattr(
asset_scanner_job,
@@ -390,6 +393,11 @@ def test_runtime_collection_reports_failed_sources_without_reconciling_them(
"_collect_ai_catalog_assets",
fake_ai_catalog,
)
monkeypatch.setattr(
asset_scanner_job,
"_collect_gitea_bundle_assets",
fake_gitea_bundle,
)
assets, relationships, prefixes, errors = asyncio.run(
asset_scanner_job._collect_runtime_assets()
@@ -636,6 +644,11 @@ def test_runtime_collection_integrates_new_asset_types_and_reconciliation_prefix
"_collect_ai_catalog_assets",
empty_collector,
)
monkeypatch.setattr(
asset_scanner_job,
"_collect_gitea_bundle_assets",
empty_collector,
)
assets, relationships, prefixes, errors = asyncio.run(
asset_scanner_job._collect_runtime_assets()
@@ -769,6 +782,71 @@ def test_ai_catalog_collector_reads_aggregates_and_model_names_only(
]
def test_gitea_bundle_readback_builds_repo_and_backup_assets() -> None:
assets, relationships = asset_scanner_job._build_gitea_bundle_assets(
{
"schema_version": "gitea_repo_bundle_backup_readback_v1",
"status": "gitea_repo_bundle_backup_readback_ready",
"host": "188",
"readback_present": True,
"ready": True,
"summary": {
"expected_repo_count": 2,
"repo_row_count": 2,
"expected_repo_missing_count": 0,
"failed_repo_count": 0,
"checksum_missing_count": 0,
"bundle_fresh": True,
"sample_restore_dry_run_ok": True,
},
"repos": [
{
"repo": "wooo/awoooi",
"present": True,
"ok": True,
"head_count": 196,
"checksum_digest_present": True,
},
{
"repo": "wooo/tsenyang-website",
"present": True,
"ok": True,
"head_count": 2,
"checksum_digest_present": True,
},
],
}
)
by_type: dict[str, list[dict]] = {}
for asset in assets:
by_type.setdefault(asset["asset_type"], []).append(asset)
assert len(by_type["gitea_repo"]) == 2
assert len(by_type["backup_target"]) == 1
assert by_type["backup_target"][0]["metadata"]["ready"] is True
assert all(
asset["metadata"]["repository_content_read_by_scanner"] is False
for asset in assets
)
assert all(
asset_scanner_job._resolve_owner_team(asset)[0] == "platform_security"
for asset in assets
)
assert relationships == [
{
"from_key": "gitea/repo/wooo/awoooi",
"to_key": "backup/gitea-bundle/188",
"relationship_type": "backs_up_to",
},
{
"from_key": "gitea/repo/wooo/tsenyang-website",
"to_key": "backup/gitea-bundle/188",
"relationship_type": "backs_up_to",
},
]
def test_scan_terminal_is_failed_when_any_collector_is_incomplete(monkeypatch) -> None:
terminal: dict[str, object] = {}