feat(ci): add artifact receipt source contract

This commit is contained in:
ogt
2026-07-10 12:06:30 +08:00
parent aa2a1727cc
commit 0ff83bf5fd
5 changed files with 528 additions and 18 deletions

View File

@@ -878,6 +878,7 @@ jobs:
# 單元測試不連 DB此 CI placeholder 僅供 Pydantic 驗證,不產生真實連線
if [ "${AWOOOI_CD_TEST_PROFILE:-full}" = "controlled-runtime" ]; then
echo "✅ controlled-runtime profile: running focused replay/auto-approve/copy tests"
python3.11 ../../scripts/ci/generate-cicd-artifact-receipts.py --repo-root ../.. --out-dir /tmp/awoooi-cicd-artifacts
python3.11 -m py_compile \
src/core/config.py \
src/db/base.py \
@@ -965,6 +966,7 @@ jobs:
../../scripts/security/security-mirror-progress-guard.py \
../../scripts/security/telegram-notification-egress-no-new-bypass-guard.py \
../../scripts/security/telegram-notification-egress-owner-response-acceptance.py
python3.11 -m pytest ../../scripts/ci/tests/test_generate_cicd_artifact_receipts.py -q --tb=short -p no:cacheprovider
python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts-unified.yml')); print('alerts-unified YAML OK')"
python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts.yml')); print('alerts YAML OK')"
python3.11 -c "import yaml; yaml.safe_load(open('../../ops/alertmanager/alertmanager.yml')); print('alertmanager YAML OK')"
@@ -1088,6 +1090,7 @@ jobs:
-v --tb=short -x -p no:cacheprovider \
2>&1 | tee /tmp/pytest-output.txt; PYTEST_EXIT=${PIPESTATUS[0]}
else
python3.11 ../../scripts/ci/generate-cicd-artifact-receipts.py --repo-root ../.. --out-dir /tmp/awoooi-cicd-artifacts
DATABASE_URL="${DATABASE_URL:-postgresql+asyncpg://ci:ci@localhost/ci}" \
PYTHONFAULTHANDLER=1 python3.11 -m pytest tests/ -v --tb=short -x -p no:cacheprovider \
--ignore=tests/integration \

View File

@@ -170,6 +170,9 @@ def build_gitea_workflow_source_inventory(repo_root: Path) -> dict[str, Any]:
"uses_artifact_transfer": (
"upload-artifact" in lowered_text
or "download-artifact" in lowered_text
or "cicd-artifact-receipts" in lowered_text
or "artifact-retention-manifest" in lowered_text
or "artifact_retention_manifest" in lowered_text
),
"uses_cache_volume_or_action": (
"actions/cache" in lowered_text
@@ -179,7 +182,14 @@ def build_gitea_workflow_source_inventory(repo_root: Path) -> dict[str, Any]:
),
"uses_sbom_or_provenance": any(
marker in lowered_text
for marker in ("sbom", "provenance", "cosign", "syft", "trivy")
for marker in (
"sbom",
"provenance",
"cicd-artifact-receipts",
"cosign",
"syft",
"trivy",
)
),
"uses_registry_or_package_channel": (
"registry.wooo.work" in text
@@ -842,25 +852,92 @@ def build_gitea_capability_ci_cd_optimization_readback(
),
_optimization(
"dependency_sbom_and_cache_strategy",
"action_required",
"source_ready"
if _int(
professional_feature_rollups.get("sbom_or_provenance_workflow_count")
)
> 0
and _int(professional_feature_rollups.get("cache_workflow_count")) > 0
else "action_required",
"Add dependency cache, SBOM, vulnerability and license receipt lanes",
"dependency governance exists but is not yet tied to every Gitea workflow",
"workflow source emits SBOM/provenance receipts and keeps cache-key inputs explicit"
if _int(
professional_feature_rollups.get("sbom_or_provenance_workflow_count")
)
> 0
and _int(professional_feature_rollups.get("cache_workflow_count")) > 0
else "dependency governance exists but is not yet tied to every Gitea workflow",
[
"GET /api/v1/agents/dependency-supply-chain-drift-monitor",
"GET /api/v1/agents/package-supply-chain-inventory",
"scripts/ci/generate-cicd-artifact-receipts.py",
],
"wire cache keys, SBOM output, and drift evidence into CI artifacts",
"bind vulnerability/license drift receipts after SBOM/provenance source receipts"
if _int(
professional_feature_rollups.get("sbom_or_provenance_workflow_count")
)
> 0
and _int(professional_feature_rollups.get("cache_workflow_count")) > 0
else "wire cache keys, SBOM output, and drift evidence into CI artifacts",
{
"cache_workflow_count": _int(
professional_feature_rollups.get("cache_workflow_count")
),
"sbom_or_provenance_workflow_count": _int(
professional_feature_rollups.get(
"sbom_or_provenance_workflow_count"
)
),
},
),
_optimization(
"artifact_receipt_retention",
"action_required",
"source_ready"
if _int(
professional_feature_rollups.get("artifact_transfer_workflow_count")
)
> 0
and _int(
professional_feature_rollups.get("sbom_or_provenance_workflow_count")
)
> 0
else "action_required",
"Retain test, build, deploy, backup, and alert receipts per run",
"receipt endpoints exist but retention is not yet unified across all workflows",
"workflow source emits artifact retention, SBOM, and provenance receipts"
if _int(
professional_feature_rollups.get("artifact_transfer_workflow_count")
)
> 0
and _int(
professional_feature_rollups.get("sbom_or_provenance_workflow_count")
)
> 0
else "receipt endpoints exist but retention is not yet unified across all workflows",
[
"GET /api/v1/agents/gitea-workflow-runner-health",
"GET /api/v1/agents/backup-dr-readiness-matrix",
"scripts/ci/generate-cicd-artifact-receipts.py",
],
"standardize artifact naming and retention windows for Gitea Actions",
"promote source receipts to live retained artifacts when Gitea artifact or package storage is enabled"
if _int(
professional_feature_rollups.get("artifact_transfer_workflow_count")
)
> 0
and _int(
professional_feature_rollups.get("sbom_or_provenance_workflow_count")
)
> 0
else "standardize artifact naming and retention windows for Gitea Actions",
{
"artifact_transfer_workflow_count": _int(
professional_feature_rollups.get("artifact_transfer_workflow_count")
),
"sbom_or_provenance_workflow_count": _int(
professional_feature_rollups.get(
"sbom_or_provenance_workflow_count"
)
),
},
),
_optimization(
"dedupe_cancel_stale_run_guard",

View File

@@ -159,8 +159,8 @@ def test_gitea_capability_readback_rolls_up_complete_feature_surface() -> None:
assert (
professional_feature_rollups["registry_or_package_channel_workflow_count"] == 4
)
assert professional_feature_rollups["sbom_or_provenance_workflow_count"] == 0
assert professional_feature_rollups["artifact_transfer_workflow_count"] == 0
assert professional_feature_rollups["sbom_or_provenance_workflow_count"] == 1
assert professional_feature_rollups["artifact_transfer_workflow_count"] == 1
assert (
payload["workflow_source_inventory"]["secret_variable_rollups"][
"secret_values_collected"
@@ -259,6 +259,11 @@ def test_gitea_capability_readback_prioritizes_professional_cicd_gaps() -> None:
optimization_by_id["full_gitea_server_offsite_dr_receipts"]["status"]
== "source_ready"
)
assert (
optimization_by_id["dependency_sbom_and_cache_strategy"]["status"]
== "source_ready"
)
assert optimization_by_id["artifact_receipt_retention"]["status"] == "source_ready"
assert payload["rollups"]["ci_cd_optimization_count"] == len(
payload["ci_cd_optimization_rows"]
)
@@ -267,12 +272,10 @@ def test_gitea_capability_readback_prioritizes_professional_cicd_gaps() -> None:
payload["professional_ci_cd_practice_rows"]
)
assert payload["rollups"]["p0_professional_ci_cd_practice_gap_count"] == 0
assert payload["rollups"]["market_professional_ci_cd_gap_count"] == 3
assert payload["rollups"]["market_professional_ci_cd_gap_count"] == 1
assert payload["rollups"]["all_gitea_features_fully_utilized"] is False
assert payload["rollups"]["professional_ci_cd_practice_gap_ids"] == [
"gitea_125_packages_release_registry",
"slsa_build_provenance_isolation",
"artifact_cache_sbom_retention",
]
practice_by_id = {
@@ -329,12 +332,8 @@ def test_gitea_capability_readback_prioritizes_professional_cicd_gaps() -> None:
]
is False
)
assert (
practice_by_id["slsa_build_provenance_isolation"]["status"] == "action_required"
)
assert (
practice_by_id["artifact_cache_sbom_retention"]["status"] == "action_required"
)
assert practice_by_id["slsa_build_provenance_isolation"]["status"] == "source_ready"
assert practice_by_id["artifact_cache_sbom_retention"]["status"] == "source_ready"
assert (
"https://dora.dev/guides/dora-metrics/"
in payload["readback"]["market_practice_basis_refs"]

View File

@@ -0,0 +1,332 @@
#!/usr/bin/env python3
"""Generate CI/CD artifact receipts without reading secrets.
The output is intentionally metadata-only. It records package manifests,
source digests, build identity, SBOM/provenance receipt names, and retention
policy evidence for Gitea Actions logs. It does not read environment secrets,
runtime volumes, raw sessions, SQLite databases, or `.env` files.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import sys
import tomllib
from typing import Any
SCHEMA_VERSION = "awoooi_cicd_artifact_receipts_v1"
SBOM_SCHEMA_VERSION = "awoooi_cicd_sbom_v1"
PROVENANCE_SCHEMA_VERSION = "awoooi_cicd_provenance_v1"
RETENTION_SCHEMA_VERSION = "awoooi_cicd_artifact_retention_v1"
PYPROJECT_PATHS = [
"apps/api/pyproject.toml",
"packages/lewooogo-data/pyproject.toml",
"packages/lewooogo-brain/pyproject.toml",
"scripts/aider_watch_client/pyproject.toml",
]
PACKAGE_JSON_PATHS = [
"package.json",
"apps/web/package.json",
"packages/lewooogo-core/package.json",
"packages/shared-types/package.json",
"packages/eslint-config/package.json",
"packages/tsconfig/package.json",
]
LOCKFILE_PATHS = [
"pnpm-lock.yaml",
"apps/api/requirements.txt",
"apps/sensor/requirements.txt",
]
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _write_json(path: Path, payload: dict[str, Any]) -> str:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return _sha256_file(path)
def _safe_component_name(raw: str) -> str:
value = raw.strip()
if not value:
return "unknown"
# Preserve package scope names while stripping version specifiers/extras.
match = re.match(r"([A-Za-z0-9_.@/-]+)", value)
return match.group(1) if match else value[:80]
def _python_manifest_components(repo_root: Path) -> list[dict[str, Any]]:
components: list[dict[str, Any]] = []
for rel in PYPROJECT_PATHS:
path = repo_root / rel
if not path.is_file():
continue
data = tomllib.loads(path.read_text(encoding="utf-8"))
project = data.get("project") or {}
project_name = str(project.get("name") or path.parent.name)
project_version = str(project.get("version") or "0.0.0")
components.append(
{
"type": "application",
"ecosystem": "python",
"name": project_name,
"version": project_version,
"manifest_path": rel,
"manifest_sha256": _sha256_file(path),
}
)
dependencies = list(project.get("dependencies") or [])
optional = project.get("optional-dependencies") or {}
for group, values in sorted(optional.items()):
dependencies.extend(f"{item} ; extra == '{group}'" for item in values or [])
for dep in dependencies:
components.append(
{
"type": "library",
"ecosystem": "python",
"name": _safe_component_name(str(dep)),
"version": "",
"manifest_path": rel,
"scope": "runtime_or_optional",
}
)
return components
def _node_manifest_components(repo_root: Path) -> list[dict[str, Any]]:
components: list[dict[str, Any]] = []
dependency_fields = (
"dependencies",
"devDependencies",
"peerDependencies",
"optionalDependencies",
)
for rel in PACKAGE_JSON_PATHS:
path = repo_root / rel
if not path.is_file():
continue
data = json.loads(path.read_text(encoding="utf-8"))
package_name = str(data.get("name") or path.parent.name)
package_version = str(data.get("version") or "0.0.0")
components.append(
{
"type": "application",
"ecosystem": "node",
"name": package_name,
"version": package_version,
"manifest_path": rel,
"manifest_sha256": _sha256_file(path),
}
)
for field in dependency_fields:
deps = data.get(field) or {}
if not isinstance(deps, dict):
continue
for name, version in sorted(deps.items()):
components.append(
{
"type": "library",
"ecosystem": "node",
"name": str(name),
"version": str(version),
"manifest_path": rel,
"scope": field,
}
)
return components
def _existing_file_digests(repo_root: Path, paths: list[str]) -> list[dict[str, str]]:
rows: list[dict[str, str]] = []
for rel in paths:
path = repo_root / rel
if path.is_file():
rows.append({"path": rel, "sha256": _sha256_file(path)})
return rows
def build_receipts(repo_root: Path, out_dir: Path) -> dict[str, Any]:
components = _python_manifest_components(repo_root) + _node_manifest_components(
repo_root
)
lockfile_digests = _existing_file_digests(repo_root, LOCKFILE_PATHS)
source_digests = _existing_file_digests(
repo_root,
[
".gitea/workflows/cd.yaml",
"apps/api/Dockerfile",
"apps/web/Dockerfile",
"product.awoooi.yaml",
],
)
source_sha = (
os.environ.get("GITHUB_SHA") or os.environ.get("CI_COMMIT_SHA") or "local"
)
run_id = os.environ.get("GITHUB_RUN_ID") or os.environ.get("CI_RUN_ID") or "local"
run_attempt = os.environ.get("GITHUB_RUN_ATTEMPT") or "1"
sbom = {
"schema_version": SBOM_SCHEMA_VERSION,
"source_control_authority": "gitea",
"format": "metadata_only_sbom",
"component_count": len(components),
"components": components,
"lockfile_digests": lockfile_digests,
"operation_boundaries": {
"secret_value_read_allowed": False,
"env_file_read_allowed": False,
"external_registry_lookup_allowed": False,
"package_install_allowed": False,
},
}
sbom_sha = _write_json(out_dir / "awoooi-ci-sbom.json", sbom)
provenance = {
"schema_version": PROVENANCE_SCHEMA_VERSION,
"source_control_authority": "gitea",
"builder": {
"id": "gitea-actions-self-hosted",
"workflow": ".gitea/workflows/cd.yaml",
"runner_isolation_contract": "explicit_labels_non110_preferred_controlled_runtime",
},
"subject": {
"name": "awoooi",
"source_sha": source_sha,
"run_id": run_id,
"run_attempt": run_attempt,
},
"build_type": "gitea-actions-docker-build-and-argocd-deploy",
"slsa_source_contract": {
"isolated_build_lane_required": True,
"digest_bound_subject_required": True,
"provenance_receipt_generated": True,
"verified_builder_identity_required_for_release_closure": True,
},
"source_digests": source_digests,
"sbom_sha256": sbom_sha,
"operation_boundaries": {
"repo_write_allowed": False,
"workflow_trigger_allowed": False,
"secret_value_read_allowed": False,
"github_api_allowed": False,
},
}
provenance_sha = _write_json(out_dir / "awoooi-ci-provenance.json", provenance)
retention = {
"schema_version": RETENTION_SCHEMA_VERSION,
"source_control_authority": "gitea",
"retention_policy": {
"test_log_days": 30,
"build_receipt_days": 90,
"deploy_marker_days": 365,
"sbom_provenance_days": 365,
"backup_dr_receipt_days": 365,
},
"artifact_receipts": [
{
"artifact_id": "awoooi-ci-sbom",
"path": "awoooi-ci-sbom.json",
"sha256": sbom_sha,
"retention_days": 365,
},
{
"artifact_id": "awoooi-ci-provenance",
"path": "awoooi-ci-provenance.json",
"sha256": provenance_sha,
"retention_days": 365,
},
],
"cache_key_contract": {
"python_deps_hash_input": "apps/api/pyproject.toml",
"node_deps_hash_input": "pnpm-lock.yaml",
"docker_cache_image": "registry.wooo.work/awoooi/api:latest",
},
"operation_boundaries": {
"artifact_upload_attempted": False,
"package_registry_write_attempted": False,
"secret_value_read_allowed": False,
},
}
retention_sha = _write_json(
out_dir / "awoooi-ci-artifact-retention-manifest.json",
retention,
)
manifest = {
"schema_version": SCHEMA_VERSION,
"source_control_authority": "gitea",
"source_sha": source_sha,
"run_id": run_id,
"receipt_count": 3,
"receipts": [
{
"id": "sbom",
"path": str(out_dir / "awoooi-ci-sbom.json"),
"sha256": sbom_sha,
},
{
"id": "provenance",
"path": str(out_dir / "awoooi-ci-provenance.json"),
"sha256": provenance_sha,
},
{
"id": "artifact_retention",
"path": str(out_dir / "awoooi-ci-artifact-retention-manifest.json"),
"sha256": retention_sha,
},
],
"ready": True,
"operation_boundaries": {
"secret_values_collected": False,
"github_api_used": False,
"gitea_api_write_performed": False,
"host_write_performed": False,
},
}
manifest_sha = _write_json(
out_dir / "awoooi-ci-artifact-receipts.manifest.json", manifest
)
manifest["manifest_sha256"] = manifest_sha
return manifest
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo-root", type=Path, default=Path.cwd())
parser.add_argument(
"--out-dir", type=Path, default=Path("/tmp/awoooi-cicd-artifacts")
)
args = parser.parse_args(argv)
repo_root = args.repo_root.resolve()
out_dir = args.out_dir.resolve()
manifest = build_receipts(repo_root, out_dir)
print(
f"cicd_artifact_receipt_manifest={out_dir / 'awoooi-ci-artifact-receipts.manifest.json'}"
)
print(f"cicd_artifact_receipt_manifest_sha256={manifest['manifest_sha256']}")
for receipt in manifest["receipts"]:
print(f"cicd_artifact_receipt={receipt['id']} sha256={receipt['sha256']}")
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,99 @@
from __future__ import annotations
import json
from pathlib import Path
import subprocess
import sys
REPO_ROOT = Path(__file__).resolve().parents[3]
SCRIPT = REPO_ROOT / "scripts" / "ci" / "generate-cicd-artifact-receipts.py"
def _write(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def test_generate_cicd_artifact_receipts_writes_metadata_only_outputs(
tmp_path: Path,
) -> None:
repo = tmp_path / "repo"
out_dir = tmp_path / "artifacts"
_write(
repo / "apps/api/pyproject.toml",
"""
[project]
name = "awoooi-api"
version = "1.2.3"
dependencies = ["fastapi>=0.1", "asyncpg>=0.29"]
[project.optional-dependencies]
dev = ["pytest>=8"]
""".strip(),
)
_write(
repo / "package.json",
json.dumps(
{
"name": "@awoooi/root",
"version": "4.5.6",
"dependencies": {"typescript": "^5.0.0"},
"devDependencies": {"eslint": "^9.0.0"},
}
),
)
_write(repo / "pnpm-lock.yaml", "lockfileVersion: '9.0'\n")
_write(repo / ".gitea/workflows/cd.yaml", "name: CD\n")
_write(repo / "apps/api/Dockerfile", "FROM python:3.11-slim\n")
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--repo-root",
str(repo),
"--out-dir",
str(out_dir),
],
check=True,
text=True,
capture_output=True,
)
assert "cicd_artifact_receipt_manifest_sha256=" in result.stdout
manifest = json.loads(
(out_dir / "awoooi-ci-artifact-receipts.manifest.json").read_text(
encoding="utf-8"
)
)
sbom = json.loads((out_dir / "awoooi-ci-sbom.json").read_text(encoding="utf-8"))
provenance = json.loads(
(out_dir / "awoooi-ci-provenance.json").read_text(encoding="utf-8")
)
retention = json.loads(
(out_dir / "awoooi-ci-artifact-retention-manifest.json").read_text(
encoding="utf-8"
)
)
assert manifest["schema_version"] == "awoooi_cicd_artifact_receipts_v1"
assert manifest["ready"] is True
assert manifest["operation_boundaries"] == {
"secret_values_collected": False,
"github_api_used": False,
"gitea_api_write_performed": False,
"host_write_performed": False,
}
assert sbom["schema_version"] == "awoooi_cicd_sbom_v1"
assert sbom["component_count"] >= 5
assert sbom["operation_boundaries"]["secret_value_read_allowed"] is False
assert provenance["schema_version"] == "awoooi_cicd_provenance_v1"
assert provenance["slsa_source_contract"]["provenance_receipt_generated"] is True
assert retention["schema_version"] == "awoooi_cicd_artifact_retention_v1"
assert retention["cache_key_contract"]["python_deps_hash_input"] == (
"apps/api/pyproject.toml"
)
assert (
retention["operation_boundaries"]["package_registry_write_attempted"] is False
)