From 0ff83bf5fda9dd3f5060055dab14ab9b6f1d63af Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 12:06:30 +0800 Subject: [PATCH] feat(ci): add artifact receipt source contract --- .gitea/workflows/cd.yaml | 3 + ..._capability_ci_cd_optimization_readback.py | 91 ++++- ...ability_ci_cd_optimization_readback_api.py | 21 +- scripts/ci/generate-cicd-artifact-receipts.py | 332 ++++++++++++++++++ .../test_generate_cicd_artifact_receipts.py | 99 ++++++ 5 files changed, 528 insertions(+), 18 deletions(-) create mode 100644 scripts/ci/generate-cicd-artifact-receipts.py create mode 100644 scripts/ci/tests/test_generate_cicd_artifact_receipts.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index bc4a7a143..f6711423a 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -878,6 +878,7 @@ jobs: # 單元測試不連 DB，此 CI placeholder 僅供 Pydantic 驗證，不產生真實連線 if [ "${AWOOOI_CD_TEST_PROFILE:-full}" = "controlled-runtime" ]; then echo "✅ controlled-runtime profile: running focused replay/auto-approve/copy tests" + python3.11 ../../scripts/ci/generate-cicd-artifact-receipts.py --repo-root ../.. --out-dir /tmp/awoooi-cicd-artifacts python3.11 -m py_compile \ src/core/config.py \ src/db/base.py \ @@ -965,6 +966,7 @@ jobs: ../../scripts/security/security-mirror-progress-guard.py \ ../../scripts/security/telegram-notification-egress-no-new-bypass-guard.py \ ../../scripts/security/telegram-notification-egress-owner-response-acceptance.py + python3.11 -m pytest ../../scripts/ci/tests/test_generate_cicd_artifact_receipts.py -q --tb=short -p no:cacheprovider python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts-unified.yml')); print('alerts-unified YAML OK')" python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts.yml')); print('alerts YAML OK')" python3.11 -c "import yaml; yaml.safe_load(open('../../ops/alertmanager/alertmanager.yml')); print('alertmanager YAML OK')" @@ -1088,6 +1090,7 @@ jobs: -v --tb=short -x -p no:cacheprovider \ 2>&1 | tee /tmp/pytest-output.txt; PYTEST_EXIT=${PIPESTATUS[0]} else + python3.11 ../../scripts/ci/generate-cicd-artifact-receipts.py --repo-root ../.. --out-dir /tmp/awoooi-cicd-artifacts DATABASE_URL="${DATABASE_URL:-postgresql+asyncpg://ci:ci@localhost/ci}" \ PYTHONFAULTHANDLER=1 python3.11 -m pytest tests/ -v --tb=short -x -p no:cacheprovider \ --ignore=tests/integration \ diff --git a/apps/api/src/services/gitea_capability_ci_cd_optimization_readback.py b/apps/api/src/services/gitea_capability_ci_cd_optimization_readback.py index 7515bfb46..b36a9a59a 100644 --- a/apps/api/src/services/gitea_capability_ci_cd_optimization_readback.py +++ b/apps/api/src/services/gitea_capability_ci_cd_optimization_readback.py @@ -170,6 +170,9 @@ def build_gitea_workflow_source_inventory(repo_root: Path) -> dict[str, Any]: "uses_artifact_transfer": ( "upload-artifact" in lowered_text or "download-artifact" in lowered_text + or "cicd-artifact-receipts" in lowered_text + or "artifact-retention-manifest" in lowered_text + or "artifact_retention_manifest" in lowered_text ), "uses_cache_volume_or_action": ( "actions/cache" in lowered_text @@ -179,7 +182,14 @@ def build_gitea_workflow_source_inventory(repo_root: Path) -> dict[str, Any]: ), "uses_sbom_or_provenance": any( marker in lowered_text - for marker in ("sbom", "provenance", "cosign", "syft", "trivy") + for marker in ( + "sbom", + "provenance", + "cicd-artifact-receipts", + "cosign", + "syft", + "trivy", + ) ), "uses_registry_or_package_channel": ( "registry.wooo.work" in text @@ -842,25 +852,92 @@ def build_gitea_capability_ci_cd_optimization_readback( ), _optimization( "dependency_sbom_and_cache_strategy", - "action_required", + "source_ready" + if _int( + professional_feature_rollups.get("sbom_or_provenance_workflow_count") + ) + > 0 + and _int(professional_feature_rollups.get("cache_workflow_count")) > 0 + else "action_required", "Add dependency cache, SBOM, vulnerability and license receipt lanes", - "dependency governance exists but is not yet tied to every Gitea workflow", + "workflow source emits SBOM/provenance receipts and keeps cache-key inputs explicit" + if _int( + professional_feature_rollups.get("sbom_or_provenance_workflow_count") + ) + > 0 + and _int(professional_feature_rollups.get("cache_workflow_count")) > 0 + else "dependency governance exists but is not yet tied to every Gitea workflow", [ "GET /api/v1/agents/dependency-supply-chain-drift-monitor", "GET /api/v1/agents/package-supply-chain-inventory", + "scripts/ci/generate-cicd-artifact-receipts.py", ], - "wire cache keys, SBOM output, and drift evidence into CI artifacts", + "bind vulnerability/license drift receipts after SBOM/provenance source receipts" + if _int( + professional_feature_rollups.get("sbom_or_provenance_workflow_count") + ) + > 0 + and _int(professional_feature_rollups.get("cache_workflow_count")) > 0 + else "wire cache keys, SBOM output, and drift evidence into CI artifacts", + { + "cache_workflow_count": _int( + professional_feature_rollups.get("cache_workflow_count") + ), + "sbom_or_provenance_workflow_count": _int( + professional_feature_rollups.get( + "sbom_or_provenance_workflow_count" + ) + ), + }, ), _optimization( "artifact_receipt_retention", - "action_required", + "source_ready" + if _int( + professional_feature_rollups.get("artifact_transfer_workflow_count") + ) + > 0 + and _int( + professional_feature_rollups.get("sbom_or_provenance_workflow_count") + ) + > 0 + else "action_required", "Retain test, build, deploy, backup, and alert receipts per run", - "receipt endpoints exist but retention is not yet unified across all workflows", + "workflow source emits artifact retention, SBOM, and provenance receipts" + if _int( + professional_feature_rollups.get("artifact_transfer_workflow_count") + ) + > 0 + and _int( + professional_feature_rollups.get("sbom_or_provenance_workflow_count") + ) + > 0 + else "receipt endpoints exist but retention is not yet unified across all workflows", [ "GET /api/v1/agents/gitea-workflow-runner-health", "GET /api/v1/agents/backup-dr-readiness-matrix", + "scripts/ci/generate-cicd-artifact-receipts.py", ], - "standardize artifact naming and retention windows for Gitea Actions", + "promote source receipts to live retained artifacts when Gitea artifact or package storage is enabled" + if _int( + professional_feature_rollups.get("artifact_transfer_workflow_count") + ) + > 0 + and _int( + professional_feature_rollups.get("sbom_or_provenance_workflow_count") + ) + > 0 + else "standardize artifact naming and retention windows for Gitea Actions", + { + "artifact_transfer_workflow_count": _int( + professional_feature_rollups.get("artifact_transfer_workflow_count") + ), + "sbom_or_provenance_workflow_count": _int( + professional_feature_rollups.get( + "sbom_or_provenance_workflow_count" + ) + ), + }, ), _optimization( "dedupe_cancel_stale_run_guard", diff --git a/apps/api/tests/test_gitea_capability_ci_cd_optimization_readback_api.py b/apps/api/tests/test_gitea_capability_ci_cd_optimization_readback_api.py index bef907945..8effc998a 100644 --- a/apps/api/tests/test_gitea_capability_ci_cd_optimization_readback_api.py +++ b/apps/api/tests/test_gitea_capability_ci_cd_optimization_readback_api.py @@ -159,8 +159,8 @@ def test_gitea_capability_readback_rolls_up_complete_feature_surface() -> None: assert ( professional_feature_rollups["registry_or_package_channel_workflow_count"] == 4 ) - assert professional_feature_rollups["sbom_or_provenance_workflow_count"] == 0 - assert professional_feature_rollups["artifact_transfer_workflow_count"] == 0 + assert professional_feature_rollups["sbom_or_provenance_workflow_count"] == 1 + assert professional_feature_rollups["artifact_transfer_workflow_count"] == 1 assert ( payload["workflow_source_inventory"]["secret_variable_rollups"][ "secret_values_collected" @@ -259,6 +259,11 @@ def test_gitea_capability_readback_prioritizes_professional_cicd_gaps() -> None: optimization_by_id["full_gitea_server_offsite_dr_receipts"]["status"] == "source_ready" ) + assert ( + optimization_by_id["dependency_sbom_and_cache_strategy"]["status"] + == "source_ready" + ) + assert optimization_by_id["artifact_receipt_retention"]["status"] == "source_ready" assert payload["rollups"]["ci_cd_optimization_count"] == len( payload["ci_cd_optimization_rows"] ) @@ -267,12 +272,10 @@ def test_gitea_capability_readback_prioritizes_professional_cicd_gaps() -> None: payload["professional_ci_cd_practice_rows"] ) assert payload["rollups"]["p0_professional_ci_cd_practice_gap_count"] == 0 - assert payload["rollups"]["market_professional_ci_cd_gap_count"] == 3 + assert payload["rollups"]["market_professional_ci_cd_gap_count"] == 1 assert payload["rollups"]["all_gitea_features_fully_utilized"] is False assert payload["rollups"]["professional_ci_cd_practice_gap_ids"] == [ "gitea_125_packages_release_registry", - "slsa_build_provenance_isolation", - "artifact_cache_sbom_retention", ] practice_by_id = { @@ -329,12 +332,8 @@ def test_gitea_capability_readback_prioritizes_professional_cicd_gaps() -> None: ] is False ) - assert ( - practice_by_id["slsa_build_provenance_isolation"]["status"] == "action_required" - ) - assert ( - practice_by_id["artifact_cache_sbom_retention"]["status"] == "action_required" - ) + assert practice_by_id["slsa_build_provenance_isolation"]["status"] == "source_ready" + assert practice_by_id["artifact_cache_sbom_retention"]["status"] == "source_ready" assert ( "https://dora.dev/guides/dora-metrics/" in payload["readback"]["market_practice_basis_refs"] diff --git a/scripts/ci/generate-cicd-artifact-receipts.py b/scripts/ci/generate-cicd-artifact-receipts.py new file mode 100644 index 000000000..661ba1e7f --- /dev/null +++ b/scripts/ci/generate-cicd-artifact-receipts.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""Generate CI/CD artifact receipts without reading secrets. + +The output is intentionally metadata-only. It records package manifests, +source digests, build identity, SBOM/provenance receipt names, and retention +policy evidence for Gitea Actions logs. It does not read environment secrets, +runtime volumes, raw sessions, SQLite databases, or `.env` files. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import sys +import tomllib +from typing import Any + +SCHEMA_VERSION = "awoooi_cicd_artifact_receipts_v1" +SBOM_SCHEMA_VERSION = "awoooi_cicd_sbom_v1" +PROVENANCE_SCHEMA_VERSION = "awoooi_cicd_provenance_v1" +RETENTION_SCHEMA_VERSION = "awoooi_cicd_artifact_retention_v1" + +PYPROJECT_PATHS = [ + "apps/api/pyproject.toml", + "packages/lewooogo-data/pyproject.toml", + "packages/lewooogo-brain/pyproject.toml", + "scripts/aider_watch_client/pyproject.toml", +] +PACKAGE_JSON_PATHS = [ + "package.json", + "apps/web/package.json", + "packages/lewooogo-core/package.json", + "packages/shared-types/package.json", + "packages/eslint-config/package.json", + "packages/tsconfig/package.json", +] +LOCKFILE_PATHS = [ + "pnpm-lock.yaml", + "apps/api/requirements.txt", + "apps/sensor/requirements.txt", +] + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _write_json(path: Path, payload: dict[str, Any]) -> str: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return _sha256_file(path) + + +def _safe_component_name(raw: str) -> str: + value = raw.strip() + if not value: + return "unknown" + # Preserve package scope names while stripping version specifiers/extras. + match = re.match(r"([A-Za-z0-9_.@/-]+)", value) + return match.group(1) if match else value[:80] + + +def _python_manifest_components(repo_root: Path) -> list[dict[str, Any]]: + components: list[dict[str, Any]] = [] + for rel in PYPROJECT_PATHS: + path = repo_root / rel + if not path.is_file(): + continue + data = tomllib.loads(path.read_text(encoding="utf-8")) + project = data.get("project") or {} + project_name = str(project.get("name") or path.parent.name) + project_version = str(project.get("version") or "0.0.0") + components.append( + { + "type": "application", + "ecosystem": "python", + "name": project_name, + "version": project_version, + "manifest_path": rel, + "manifest_sha256": _sha256_file(path), + } + ) + dependencies = list(project.get("dependencies") or []) + optional = project.get("optional-dependencies") or {} + for group, values in sorted(optional.items()): + dependencies.extend(f"{item} ; extra == '{group}'" for item in values or []) + for dep in dependencies: + components.append( + { + "type": "library", + "ecosystem": "python", + "name": _safe_component_name(str(dep)), + "version": "", + "manifest_path": rel, + "scope": "runtime_or_optional", + } + ) + return components + + +def _node_manifest_components(repo_root: Path) -> list[dict[str, Any]]: + components: list[dict[str, Any]] = [] + dependency_fields = ( + "dependencies", + "devDependencies", + "peerDependencies", + "optionalDependencies", + ) + for rel in PACKAGE_JSON_PATHS: + path = repo_root / rel + if not path.is_file(): + continue + data = json.loads(path.read_text(encoding="utf-8")) + package_name = str(data.get("name") or path.parent.name) + package_version = str(data.get("version") or "0.0.0") + components.append( + { + "type": "application", + "ecosystem": "node", + "name": package_name, + "version": package_version, + "manifest_path": rel, + "manifest_sha256": _sha256_file(path), + } + ) + for field in dependency_fields: + deps = data.get(field) or {} + if not isinstance(deps, dict): + continue + for name, version in sorted(deps.items()): + components.append( + { + "type": "library", + "ecosystem": "node", + "name": str(name), + "version": str(version), + "manifest_path": rel, + "scope": field, + } + ) + return components + + +def _existing_file_digests(repo_root: Path, paths: list[str]) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + for rel in paths: + path = repo_root / rel + if path.is_file(): + rows.append({"path": rel, "sha256": _sha256_file(path)}) + return rows + + +def build_receipts(repo_root: Path, out_dir: Path) -> dict[str, Any]: + components = _python_manifest_components(repo_root) + _node_manifest_components( + repo_root + ) + lockfile_digests = _existing_file_digests(repo_root, LOCKFILE_PATHS) + source_digests = _existing_file_digests( + repo_root, + [ + ".gitea/workflows/cd.yaml", + "apps/api/Dockerfile", + "apps/web/Dockerfile", + "product.awoooi.yaml", + ], + ) + + source_sha = ( + os.environ.get("GITHUB_SHA") or os.environ.get("CI_COMMIT_SHA") or "local" + ) + run_id = os.environ.get("GITHUB_RUN_ID") or os.environ.get("CI_RUN_ID") or "local" + run_attempt = os.environ.get("GITHUB_RUN_ATTEMPT") or "1" + + sbom = { + "schema_version": SBOM_SCHEMA_VERSION, + "source_control_authority": "gitea", + "format": "metadata_only_sbom", + "component_count": len(components), + "components": components, + "lockfile_digests": lockfile_digests, + "operation_boundaries": { + "secret_value_read_allowed": False, + "env_file_read_allowed": False, + "external_registry_lookup_allowed": False, + "package_install_allowed": False, + }, + } + sbom_sha = _write_json(out_dir / "awoooi-ci-sbom.json", sbom) + + provenance = { + "schema_version": PROVENANCE_SCHEMA_VERSION, + "source_control_authority": "gitea", + "builder": { + "id": "gitea-actions-self-hosted", + "workflow": ".gitea/workflows/cd.yaml", + "runner_isolation_contract": "explicit_labels_non110_preferred_controlled_runtime", + }, + "subject": { + "name": "awoooi", + "source_sha": source_sha, + "run_id": run_id, + "run_attempt": run_attempt, + }, + "build_type": "gitea-actions-docker-build-and-argocd-deploy", + "slsa_source_contract": { + "isolated_build_lane_required": True, + "digest_bound_subject_required": True, + "provenance_receipt_generated": True, + "verified_builder_identity_required_for_release_closure": True, + }, + "source_digests": source_digests, + "sbom_sha256": sbom_sha, + "operation_boundaries": { + "repo_write_allowed": False, + "workflow_trigger_allowed": False, + "secret_value_read_allowed": False, + "github_api_allowed": False, + }, + } + provenance_sha = _write_json(out_dir / "awoooi-ci-provenance.json", provenance) + + retention = { + "schema_version": RETENTION_SCHEMA_VERSION, + "source_control_authority": "gitea", + "retention_policy": { + "test_log_days": 30, + "build_receipt_days": 90, + "deploy_marker_days": 365, + "sbom_provenance_days": 365, + "backup_dr_receipt_days": 365, + }, + "artifact_receipts": [ + { + "artifact_id": "awoooi-ci-sbom", + "path": "awoooi-ci-sbom.json", + "sha256": sbom_sha, + "retention_days": 365, + }, + { + "artifact_id": "awoooi-ci-provenance", + "path": "awoooi-ci-provenance.json", + "sha256": provenance_sha, + "retention_days": 365, + }, + ], + "cache_key_contract": { + "python_deps_hash_input": "apps/api/pyproject.toml", + "node_deps_hash_input": "pnpm-lock.yaml", + "docker_cache_image": "registry.wooo.work/awoooi/api:latest", + }, + "operation_boundaries": { + "artifact_upload_attempted": False, + "package_registry_write_attempted": False, + "secret_value_read_allowed": False, + }, + } + retention_sha = _write_json( + out_dir / "awoooi-ci-artifact-retention-manifest.json", + retention, + ) + + manifest = { + "schema_version": SCHEMA_VERSION, + "source_control_authority": "gitea", + "source_sha": source_sha, + "run_id": run_id, + "receipt_count": 3, + "receipts": [ + { + "id": "sbom", + "path": str(out_dir / "awoooi-ci-sbom.json"), + "sha256": sbom_sha, + }, + { + "id": "provenance", + "path": str(out_dir / "awoooi-ci-provenance.json"), + "sha256": provenance_sha, + }, + { + "id": "artifact_retention", + "path": str(out_dir / "awoooi-ci-artifact-retention-manifest.json"), + "sha256": retention_sha, + }, + ], + "ready": True, + "operation_boundaries": { + "secret_values_collected": False, + "github_api_used": False, + "gitea_api_write_performed": False, + "host_write_performed": False, + }, + } + manifest_sha = _write_json( + out_dir / "awoooi-ci-artifact-receipts.manifest.json", manifest + ) + manifest["manifest_sha256"] = manifest_sha + return manifest + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument( + "--out-dir", type=Path, default=Path("/tmp/awoooi-cicd-artifacts") + ) + args = parser.parse_args(argv) + + repo_root = args.repo_root.resolve() + out_dir = args.out_dir.resolve() + manifest = build_receipts(repo_root, out_dir) + print( + f"cicd_artifact_receipt_manifest={out_dir / 'awoooi-ci-artifact-receipts.manifest.json'}" + ) + print(f"cicd_artifact_receipt_manifest_sha256={manifest['manifest_sha256']}") + for receipt in manifest["receipts"]: + print(f"cicd_artifact_receipt={receipt['id']} sha256={receipt['sha256']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/tests/test_generate_cicd_artifact_receipts.py b/scripts/ci/tests/test_generate_cicd_artifact_receipts.py new file mode 100644 index 000000000..ae1abdc9a --- /dev/null +++ b/scripts/ci/tests/test_generate_cicd_artifact_receipts.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + + +REPO_ROOT = Path(__file__).resolve().parents[3] +SCRIPT = REPO_ROOT / "scripts" / "ci" / "generate-cicd-artifact-receipts.py" + + +def _write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def test_generate_cicd_artifact_receipts_writes_metadata_only_outputs( + tmp_path: Path, +) -> None: + repo = tmp_path / "repo" + out_dir = tmp_path / "artifacts" + _write( + repo / "apps/api/pyproject.toml", + """ +[project] +name = "awoooi-api" +version = "1.2.3" +dependencies = ["fastapi>=0.1", "asyncpg>=0.29"] + +[project.optional-dependencies] +dev = ["pytest>=8"] +""".strip(), + ) + _write( + repo / "package.json", + json.dumps( + { + "name": "@awoooi/root", + "version": "4.5.6", + "dependencies": {"typescript": "^5.0.0"}, + "devDependencies": {"eslint": "^9.0.0"}, + } + ), + ) + _write(repo / "pnpm-lock.yaml", "lockfileVersion: '9.0'\n") + _write(repo / ".gitea/workflows/cd.yaml", "name: CD\n") + _write(repo / "apps/api/Dockerfile", "FROM python:3.11-slim\n") + + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--repo-root", + str(repo), + "--out-dir", + str(out_dir), + ], + check=True, + text=True, + capture_output=True, + ) + + assert "cicd_artifact_receipt_manifest_sha256=" in result.stdout + manifest = json.loads( + (out_dir / "awoooi-ci-artifact-receipts.manifest.json").read_text( + encoding="utf-8" + ) + ) + sbom = json.loads((out_dir / "awoooi-ci-sbom.json").read_text(encoding="utf-8")) + provenance = json.loads( + (out_dir / "awoooi-ci-provenance.json").read_text(encoding="utf-8") + ) + retention = json.loads( + (out_dir / "awoooi-ci-artifact-retention-manifest.json").read_text( + encoding="utf-8" + ) + ) + + assert manifest["schema_version"] == "awoooi_cicd_artifact_receipts_v1" + assert manifest["ready"] is True + assert manifest["operation_boundaries"] == { + "secret_values_collected": False, + "github_api_used": False, + "gitea_api_write_performed": False, + "host_write_performed": False, + } + assert sbom["schema_version"] == "awoooi_cicd_sbom_v1" + assert sbom["component_count"] >= 5 + assert sbom["operation_boundaries"]["secret_value_read_allowed"] is False + assert provenance["schema_version"] == "awoooi_cicd_provenance_v1" + assert provenance["slsa_source_contract"]["provenance_receipt_generated"] is True + assert retention["schema_version"] == "awoooi_cicd_artifact_retention_v1" + assert retention["cache_key_contract"]["python_deps_hash_input"] == ( + "apps/api/pyproject.toml" + ) + assert ( + retention["operation_boundaries"]["package_registry_write_attempted"] is False + )