# =============================================================================
# AWOOOI CD Pipeline (Gitea Actions - 方案 B)
# =============================================================================
# 流程: Build → Push to Harbor → Deploy to K8s
# 加速措施:
# 1. Docker Layer Cache → Harbor registry cache
# 2. 內部 Mirror → 192.168.0.110:5001 (Harbor Proxy Cache for DockerHub)
# 3. 非 110 runner 的 Docker pull/push 走 registry.wooo.work HTTPS alias，
# 避免要求 runner root 修改 insecure registry；K8s image pull 仍保留
# 192.168.0.110:5000 內網 Harbor route。
# 2026-03-29 Claude Code (ADR-039) - Retry after creating Harbor project
name: CD Pipeline
on:
# 2026-06-29 Codex: restore main push CD only after the non-110
# awoooi-non110-* runner lane read back registration metadata, active
# service, capacity=1, pressure OK, rollback unit, and label target-match.
# 110 incident runner labels and generic labels remain fail-closed via
# ops/runner/guard-gitea-runner-pressure.py.
push:
branches:
- main
workflow_dispatch:
# 手動觸發永遠可用（用於補跑、緊急部署）
# Main deployments run serially so concurrent writers cannot starve production closure.
# Job timeouts, rollout verification, and deploy markers terminate or expose failed runs.
concurrency:
group: cd-deploy-${{ github.ref }}
cancel-in-progress: false
env:
HARBOR: registry.wooo.work
# Harbor Proxy Cache (指向 DockerHub 的內部 Mirror，避免拉取限額)
HARBOR_MIRROR: 192.168.0.110:5001
# OTEL CI/CD 監控 (2026-03-31 #46c - 遷移到 Gitea)
OTEL_EXPORTER_OTLP_ENDPOINT: http://192.168.0.188:24318
OTEL_SERVICE_NAME: awoooi-cd
OTEL_RESOURCE_ATTRIBUTES: service.version=${{ github.sha }},deployment.environment=production
CI_IMAGE: registry.wooo.work/awoooi/ci-runner:act-22.04
# 2026-06-28 Codex: 110 runner pressure is an incident-grade capacity guard.
# Do not flip this to warn-only until non-110 readiness is verified.
HOST_WEB_BUILD_PRESSURE_WARN_ONLY: "0"
# 2026-06-30 Codex: CD is now pinned to the dedicated awoooi-non110-host lane.
# Keep the pressure guard fail-hard, but allow normal below-saturation load on
# that dedicated lane so P0 deploys are not stuck behind load5/core 0.85-1.05.
HOST_WEB_BUILD_PRESSURE_MAX_LOAD5_PER_CORE: "1.05"
# Docker lock contention is also fail-hard during the same incident window.
DOCKER_BUILD_LOCK_WARN_ONLY: "0"
# 2026-05-24 Codex: deploy through the currently Ready control-plane node.
# 120 is NotReady/SchedulingDisabled and its SSH/API endpoints are currently
# unreachable; pinning CD to it blocks secret injection before GitOps deploy.
K8S_SSH_HOST: 192.168.0.121
K8S_API_SERVER: https://192.168.0.121:6443
# 2026-06-01 Codex: post-deploy health/smoke probes use the production
# public API. The old 192.168.0.125 NodePort VIP can be absent while the
# public route and in-cluster service are healthy, causing false failures.
API_HEALTH_URL: https://awoooi.wooo.work/api/v1/health
ALERT_CHAIN_API_URL: https://awoooi.wooo.work
jobs:
workflow-shape:
# 2026-06-28 Codex: Gitea 1.25 may mark a workflow invalid when every
# root job has a job-level `if`. Keep one no-op root job without `if` so
# cd.yaml stays parseable while deploy jobs remain guarded below.
runs-on: awoooi-non110-host
timeout-minutes: 1
steps:
- name: Confirm CD Workflow Shape
run: echo "cd.yaml root job present; deploy jobs remain guarded."
cancel-stale-cd:
# 2026-06-28 Codex: keep a visible no-op run for controlled queue
# cancellation. If every job is skipped, Gitea may not create a run and
# the stale pre-guard CD queue is not superseded by concurrency.
if: ${{ github.event_name == 'push' && contains(github.event.head_commit.message, 'cancel-stale-cd') }}
runs-on: awoooi-non110-host
timeout-minutes: 3
steps:
- name: Confirm Stale CD Queue Cancellation
run: |
echo "cancel-stale-cd marker accepted; deploy jobs are intentionally skipped."
tests:
# 2026-06-28 Codex: Gitea does not consistently short-circuit `[skip ci]`
# on CD-generated deploy commits. Skip jobs explicitly so marker commits
# do not trigger a self-feeding CD loop; `cancel-stale-cd` is a
# controlled no-op trigger used only to cancel stale pre-guard runs.
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd')) }}
# 2026-04-30 Codex: run the tests job on the host runner and launch the
# CI image explicitly. The act-managed job container can disappear mid-test
# with Docker RWLayer=nil on the shared 110 daemon.
timeout-minutes: 30
runs-on: awoooi-non110-host
# 2026-04-10 ogt: B5 改用 docker run 本地啟動，移除 services: 宣告
# Gitea act runner 的 services: container name 為空，導致 CI 失敗
steps:
- name: Bootstrap Host Runner Tools
# 2026-06-28 Codex: awoooi-non110-host maps to the dedicated
# non-110 runner lane. Bootstrap tools defensively because host
# runners can start without the CI toolchain preinstalled.
run: |
if command -v apk >/dev/null 2>&1; then
apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
fi
- name: Checkout source from Gitea
env:
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
run: |
set -euo pipefail
git init .
git remote remove origin 2>/dev/null || true
git remote add origin "$GITEA_SOURCE_URL"
git fetch --no-tags --prune --depth=1 origin "$GITHUB_SHA"
git checkout --force --detach FETCH_HEAD
- name: Wait for Host Web Build Pressure
# 2026-06-28 Codex: 110 runner pressure remains incident-grade and
# fail-hard until runner work is moved or hard-limited.
run: bash scripts/ci/wait-host-web-build-pressure.sh
- name: Guard Workflow Secret Surfaces
run: node scripts/ci/check-gitea-step-env-secrets.js
# 2026-03-31 ogt: 優化告警格式 - 提高可讀性
- name: Get Commit Info
id: commit
run: |
echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
echo "message=$(git log -1 --pretty=%s | head -c 50)" >> $GITHUB_OUTPUT
echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT
- name: Notify Pipeline Start
# 2026-04-16 ogt + Claude Sonnet 4.6: 改用 HTML 結構化格式，提升可讀性
env:
TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
run: |
COMMIT_MSG="${{ steps.commit.outputs.message }}"
SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
ACTOR="${{ github.actor }}"
# HTML escape commit message（防特殊字元破壞 HTML）
COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&/g; s//\>/g')
MSG=$(printf '🚀 AWOOOI 部署開始\n├ 📝 %s\n├ 🔖 %s\n└ 👤 %s' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}")
# 2026-05-02 Claude Opus 4.7 + 統帥 ogt: notify 失敗不該擋整條 CI（鐵證:
# curl 400 從 5/1 起連續炸 14 個 commit 的 build-and-deploy）— 對齊 line 922 既有 pattern
if AWOOI_CICD_STATUS=running \
AWOOI_CICD_STAGE=tests \
AWOOI_CICD_JOB_NAME="AWOOOI 部署開始" \
AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \
scripts/ci/notify-awoooi-cicd.sh; then
echo "✅ CI/CD start notification mirrored through AWOOI API"
else
echo "AWOOI API notify failed; direct Telegram fallback disabled to preserve AwoooP receipt chain"
fi
# 2026-03-31 ogt: Phase 22.0 CI 測試 (禁止 Mock - feedback_no_mock_testing.md)
# 2026-04-01 ogt: 持久化 venv 加速 - /opt/api-venv 跨 run 保留
# pyproject.toml hash 變才重裝，其餘直接 activate (節省 ~6-7 min)
- name: Run API Tests
run: |
CHANGED_FILES=""
if [ -r "${GITHUB_EVENT_PATH:-}" ]; then
CHANGED_FILES="$(python3 - <<'PY'
import json
import os
event_path = os.environ.get("GITHUB_EVENT_PATH")
files = []
with open(event_path, "r", encoding="utf-8") as handle:
payload = json.load(handle)
for commit in payload.get("commits", []) or []:
for key in ("added", "modified", "removed"):
files.extend(commit.get(key, []) or [])
for path in dict.fromkeys(files):
print(path)
PY
)"
fi
if [ -z "$CHANGED_FILES" ]; then
BASE_SHA="${{ github.event.before }}"
if [ -n "$BASE_SHA" ] && ! printf '%s' "$BASE_SHA" | grep -Eq '^0+$'; then
git fetch --no-tags --depth=50 origin "${GITHUB_REF_NAME:-main}" >/dev/null 2>&1 || true
if git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then
CHANGED_FILES="$(git diff --name-only "$BASE_SHA" "${GITHUB_SHA:-HEAD}")"
fi
fi
fi
if [ -z "$CHANGED_FILES" ]; then
CHANGED_FILES="$(git show --format= --name-only --no-renames HEAD)"
fi
printf 'CD changed files:\n%s\n' "$CHANGED_FILES"
CONTROLLED_RUNTIME_TEST_PROFILE=1
while IFS= read -r changed_file; do
[ -z "$changed_file" ] && continue
case "$changed_file" in
# 2026-06-29 Codex: UI-only changes are verified by the
# frontend build in build-and-deploy. Keep them on the narrow
# profile so non-110 CD does not run B5's Docker/socket DB
# integration for copy/layout fixes.
apps/web/*)
;;
.gitea/workflows/cd.yaml)
;;
# 2026-06-30 Codex: workflow secret-transport and guard-only
# hardening must stay on the narrow profile. These changes are
# validated by workflow-shape, the secret-surface guard, and the
# runner pressure/profile tests; sending them to full/B5 would
# reintroduce the heavy runner path while not increasing coverage.
.gitea/workflows/cd-dev.yaml)
;;
.gitea/workflows/code-review.yaml)
;;
.gitea/workflows/deploy-alerts.yaml)
;;
.gitea/workflows/e2e-health.yaml)
;;
.gitea/workflows/ansible-lint.yml)
;;
.gitea/workflows/harbor-110-local-repair.yaml)
;;
.gitea/workflows/run-migration.yml)
;;
scripts/ci/check-gitea-step-env-secrets.js)
;;
# 2026-06-29 Codex: the onboarding warning-step workflow is
# copied in a disabled workflow_dispatch-only state. Treat the
# source and template files as controlled-runtime sources so the
# CD lane does not fall into full/B5 just for placing the inert
# guarded workflow shell.
.gitea/workflows/awoooi-onboarding-warning-step.yaml)
;;
docs/operations/templates/awoooi-gitea-onboarding-warning-step.workflow.yaml)
;;
# 2026-06-29 Codex: build-and-deploy writes only these GitOps
# deploy marker files after image push. A later merge commit can
# carry them back through CD; keep that marker merge on the
# controlled profile so non-110 CD does not fall into B5's Docker
# socket path just because the previous deploy recorded image
# truth.
k8s/awoooi-prod/04-configmap.yaml)
;;
k8s/awoooi-prod/06-deployment-api.yaml)
;;
k8s/awoooi-prod/07-rbac.yaml)
;;
k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml)
;;
k8s/awoooi-prod/08-deployment-worker.yaml)
;;
k8s/awoooi-prod/10-deployment-auto-repair-canary.yaml)
;;
k8s/awoooi-prod/kustomization.yaml)
;;
# 2026-07-02 Codex: public maintenance fallback Nginx
# source and static assets are verified by source contracts,
# shell syntax, and focused reboot tests. Keep them off B5's
# Docker/socket DB path so edge 502 hardening is not blocked by
# an unrelated full-profile integration lane.
k8s/nginx/awoooi-prod.conf)
;;
ops/maintenance/maintenance.html)
;;
ops/maintenance/nginx-502-maintenance-snippet.conf)
;;
ops/nginx/awoooi.wooo.work.conf)
;;
scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh)
;;
scripts/reboot-recovery/public-maintenance-fallback-probe.py)
;;
product.awoooi.yaml)
;;
docs/LOGBOOK.md)
;;
docs/runbooks/REBOOT-RECOVERY-SOP.md)
;;
docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md)
;;
docs/runbooks/FULL-STACK-COLD-START-SOP.md)
;;
docs/runbooks/HOST-RUNAWAY-PROCESS-AIOPS-PLAYBOOK.md)
;;
docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md)
;;
docs/awooop/TELEGRAM-INCIDENT-NOTIFICATION-MODEL.md)
;;
docs/security/TELEGRAM-NOTIFICATION-EGRESS-INVENTORY.md)
;;
docs/security/TELEGRAM-NOTIFICATION-EGRESS-MIGRATION-PLAN-DRAFT.md)
;;
docs/security/TELEGRAM-NOTIFICATION-EGRESS-NO-NEW-BYPASS-GUARD.md)
;;
docs/security/TELEGRAM-NOTIFICATION-EGRESS-OWNER-REQUEST-DRAFT.md)
;;
docs/security/TELEGRAM-NOTIFICATION-EGRESS-OWNER-RESPONSE-ACCEPTANCE.md)
;;
docs/security/telegram-notification-egress-inventory.snapshot.json)
;;
docs/security/telegram-notification-egress-owner-request-draft.snapshot.json)
;;
docs/security/telegram-notification-egress-migration-plan-draft.snapshot.json)
;;
docs/security/telegram-notification-egress-no-new-bypass-guard.snapshot.json)
;;
docs/security/telegram-notification-egress-owner-response-acceptance.snapshot.json)
;;
docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md)
;;
docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md)
;;
docs/schemas/product_awoooi_manifest_v1.schema.json)
;;
docs/operations/product-awoooi-manifest-standard.snapshot.json)
;;
docs/operations/awoooi-priority-work-order-readback.snapshot.json)
;;
docs/operations/awooop-conversation-event-hot-path-index-apply-receipt-*.snapshot.json)
;;
docs/operations/awoooi-credential-escrow-evidence-controlled-closeout-receipt.snapshot.json)
;;
docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json)
;;
docs/operations/awoooi-gitea-private-inventory-p0-scorecard.snapshot.json)
;;
docs/operations/awoooi-gitea-private-inventory-controlled-closeout-receipt.snapshot.json)
;;
docs/operations/awoooi-gitea-authenticated-inventory-payload-validation.snapshot.json)
;;
docs/security/GITEA-REPO-INVENTORY-SNAPSHOT.md)
;;
docs/security/gitea-repo-inventory.snapshot.json)
;;
docs/operations/p0-cicd-baseline-source-readiness.snapshot.json)
;;
docs/operations/awoooi-gitea-onboarding-warning-step-template-copy-receipt.snapshot.json)
;;
.gitea/workflows/awoooi-onboarding-warning-step.yaml)
;;
docs/operations/templates/awoooi-gitea-onboarding-warning-step.workflow.yaml)
;;
docs/operations/awoooi-production-deploy-readback-blocker.snapshot.json)
;;
docs/evaluations/backup_dr_target_inventory_2026-06-04.json)
;;
docs/evaluations/backup_dr_readiness_matrix_2026-06-04.json)
;;
docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json)
;;
apps/api/src/api/v1/agents.py)
;;
apps/api/src/api/v1/iwooos.py)
;;
apps/api/src/api/v1/webhooks.py)
;;
apps/api/tests/test_alertmanager_webhook_metrics.py)
;;
apps/api/src/core/config.py)
;;
apps/api/src/db/base.py)
;;
apps/api/src/services/agent_replay_normalizer.py)
;;
apps/api/src/services/ai_agent_log_intelligence_integration_readback.py)
;;
apps/api/src/services/ai_agent_log_feedback_receipt_dry_run.py)
;;
apps/api/src/services/ai_agent_log_post_write_verifier_dry_run.py)
;;
apps/api/src/services/ai_agent_log_controlled_writeback_plan_readback.py)
;;
apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py)
;;
apps/api/src/services/ai_agent_log_controlled_writeback_dispatch.py)
;;
apps/api/src/services/ai_agent_log_controlled_writeback_consumer_readback.py)
;;
apps/api/src/services/ai_agent_log_controlled_writeback_consumer_apply.py)
;;
apps/api/src/services/ai_agent_autonomous_runtime_control.py)
;;
apps/api/src/services/ai_agent_report_truth_actionability_review.py)
;;
apps/api/src/services/awooop_ansible_audit_service.py)
;;
apps/api/src/services/awooop_ansible_check_mode_service.py)
;;
apps/api/src/services/executor_trust_boundary_readback.py)
;;
apps/api/src/workers/ansible_executor_broker.py)
;;
apps/api/src/workers/signal_worker.py)
;;
apps/api/migrations/adr090e_ansible_learning_writeback_operation_type.sql)
;;
apps/api/migrations/adr090e_ansible_learning_writeback_operation_type_down.sql)
;;
apps/api/migrations/adr090f_log_controlled_writeback_dispatch_operation_type.sql)
;;
apps/api/migrations/adr090f_log_controlled_writeback_dispatch_operation_type_down.sql)
;;
apps/api/migrations/asset_inventory_database_types_2026-07-14.sql)
;;
apps/api/migrations/asset_inventory_database_types_2026-07-14_down.sql)
;;
apps/api/src/services/auto_approve.py)
;;
apps/api/src/services/decision_fusion.py)
;;
apps/api/src/services/heartbeat_report_service.py)
;;
apps/api/src/services/credential_escrow_evidence_intake_readiness.py)
;;
apps/api/src/services/gitea_authenticated_inventory_payload_validation.py)
;;
apps/api/src/services/gitea_owner_coverage_attestation_validation.py)
;;
apps/api/src/services/gitea_private_inventory_closeout_validation.py)
;;
apps/api/src/services/gitea_private_inventory_p0_scorecard.py)
;;
apps/api/src/services/gitea_workflow_runner_owner_attestation_request.py)
;;
apps/api/src/services/gitea_cicd_alert_receipt_chain_readback.py)
;;
apps/api/src/services/gitea_capability_ci_cd_optimization_readback.py)
;;
apps/api/src/services/reboot_auto_recovery_slo_scorecard.py)
;;
apps/api/src/services/windows99_vmx_source_locator_readback.py)
;;
apps/api/src/services/reboot_auto_recovery_drill_preflight.py)
;;
apps/api/src/services/stockplatform_public_api_runtime_readback.py)
;;
apps/api/src/services/stockplatform_public_api_controlled_recovery_preflight.py)
;;
apps/api/src/services/harbor_registry_controlled_recovery_preflight.py)
;;
apps/api/src/services/harbor_registry_controlled_recovery_receipt.py)
;;
apps/api/src/services/iwooos_security_operating_system.py)
;;
apps/api/Dockerfile)
;;
apps/api/src/services/awoooi_gitea_onboarding_warning_step_dashboard.py)
;;
apps/api/src/services/awoooi_gitea_onboarding_warning_step_owner_package.py)
;;
apps/api/src/services/awoooi_gitea_onboarding_warning_step_owner_response_preflight.py)
;;
apps/api/src/services/awoooi_gitea_onboarding_warning_step_template_copy_apply_gate.py)
;;
apps/api/src/services/awoooi_gitea_onboarding_warning_step_template_copy_execution_plan.py)
;;
apps/api/src/services/awoooi_gitea_onboarding_warning_step_template_copy_receipt.py)
;;
apps/api/src/services/awoooi_gitea_onboarding_warning_step_runtime_enablement_gate.py)
;;
apps/api/src/services/awoooi_new_product_onboarding_page_model.py)
;;
apps/api/src/services/awoooi_onboarding_reminder_contract.py)
;;
apps/api/src/services/awoooi_onboarding_source_contracts.py)
;;
apps/api/src/services/awoooi_priority_work_order_readback.py)
;;
apps/api/src/services/awoooi_product_onboarding_guard.py)
;;
apps/api/src/services/p0_cicd_baseline_source_readiness.py)
;;
apps/api/src/services/product_awoooi_manifest_standard.py)
;;
apps/api/src/api/v1/platform/events.py)
;;
apps/api/src/api/v1/platform/operator_runs.py)
;;
apps/api/src/jobs/ai_slo_watchdog_job.py)
;;
apps/api/src/repositories/knowledge_repository.py)
;;
apps/api/src/services/knowledge_service.py)
;;
apps/api/src/models/knowledge.py)
;;
apps/api/src/models/playbook.py)
;;
apps/api/src/services/auto_repair_service.py)
;;
apps/api/src/services/awoooi_production_deploy_readback_blocker.py)
;;
apps/api/src/services/backup_dr_target_inventory.py)
;;
apps/api/src/services/backup_dr_readiness_matrix.py)
;;
apps/api/src/services/gitea_repo_bundle_backup_readback.py)
;;
apps/api/src/services/channel_hub.py)
;;
apps/api/src/services/decision_manager.py)
;;
apps/api/src/services/delivery_closure_workbench.py)
;;
apps/api/src/services/platform_operator_service.py)
;;
apps/api/src/services/telegram_alert_ai_automation_matrix.py)
;;
apps/api/src/services/telegram_alert_monitoring_coverage_readback.py)
;;
apps/api/src/services/telegram_gateway.py)
;;
apps/api/tests/test_agent_replay_normalizer.py)
;;
apps/api/tests/test_ai_agent_log_intelligence_integration_readback_api.py)
;;
apps/api/tests/test_ai_agent_log_feedback_receipt_dry_run_api.py)
;;
apps/api/tests/test_ai_agent_log_post_write_verifier_dry_run_api.py)
;;
apps/api/tests/test_ai_agent_log_controlled_writeback_plan_readback_api.py)
;;
apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py)
;;
apps/api/tests/test_ai_agent_log_controlled_writeback_dispatch_api.py)
;;
apps/api/tests/test_ai_agent_log_controlled_writeback_consumer_readback_api.py)
;;
apps/api/tests/test_ai_agent_log_controlled_writeback_consumer_apply_api.py)
;;
apps/api/tests/test_ai_agent_autonomous_runtime_control.py)
;;
apps/api/tests/test_ai_agent_report_truth_actionability_review.py)
;;
apps/api/tests/test_ai_agent_report_truth_actionability_review_api.py)
;;
apps/api/tests/test_awooop_truth_chain_service.py)
;;
apps/api/tests/test_executor_trust_boundary_readback.py)
;;
apps/api/tests/test_signal_worker_ansible_executor_binding.py)
;;
apps/api/tests/test_channel_hub_grouped_alert_events.py)
;;
apps/api/tests/test_shadow_auto_approve.py)
;;
apps/api/tests/test_destructive_patterns.py)
;;
apps/api/tests/test_approval_pending_visibility.py)
;;
apps/api/tests/test_awooop_operator_timeline_labels.py)
;;
apps/api/tests/test_config_url_validation.py)
;;
apps/api/tests/test_delivery_closure_workbench_api.py)
;;
apps/api/tests/test_runtime_bootstrap_guards.py)
;;
apps/api/tests/test_backup_dr_target_inventory.py)
;;
apps/api/tests/test_backup_dr_target_inventory_api.py)
;;
apps/api/tests/test_backup_dr_readiness_matrix.py)
;;
apps/api/tests/test_backup_dr_readiness_matrix_api.py)
;;
apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py)
;;
apps/api/tests/test_gitea_cicd_alert_receipt_chain_readback_api.py)
;;
apps/api/tests/test_gitea_capability_ci_cd_optimization_readback_api.py)
;;
apps/api/tests/test_knowledge_repository_read_model.py)
;;
apps/api/tests/test_credential_escrow_evidence_intake_readiness_api.py)
;;
apps/api/tests/test_gitea_private_inventory_p0_scorecard_api.py)
;;
apps/api/tests/test_gitea_workflow_runner_owner_attestation_request_api.py)
;;
apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py)
;;
apps/api/tests/test_windows99_vmx_source_locator_readback_api.py)
;;
apps/api/tests/test_stockplatform_public_api_runtime_readback.py)
;;
apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py)
;;
apps/api/tests/test_harbor_registry_controlled_recovery_preflight.py)
;;
apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py)
;;
apps/api/tests/test_iwooos_security_operating_system.py)
;;
apps/api/tests/test_iwooos_wazuh_prod_manifest.py)
;;
apps/api/tests/test_awoooi_production_deploy_readback_blocker.py)
;;
apps/api/tests/test_awoooi_priority_work_order_readback_api.py)
;;
apps/api/tests/test_telegram_alert_ai_automation_matrix_api.py)
;;
apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py)
;;
apps/api/tests/e2e_network_test.py)
;;
apps/api/tests/test_p0_cicd_baseline_source_readiness_api.py)
;;
apps/api/tests/test_product_awoooi_manifest_standard_api.py)
;;
apps/api/tests/test_trust_drift_watchdog.py)
;;
apps/web/src/app/\[locale\]/governance/tabs/events-tab.tsx)
;;
apps/web/src/app/\[locale\]/governance/tabs/queue-tab.tsx)
;;
apps/web/src/app/\[locale\]/governance/tabs/slo-tab.tsx)
;;
ops/runner/read-public-gitea-actions-queue.py)
;;
ops/runner/README.md)
;;
ops/runner/check-awoooi-non110-runner-readiness.sh)
;;
ops/runner/install-awoooi-non110-runner-user-service.sh)
;;
ops/runner/register-awoooi-110-controlled-cd-lane-drain.sh)
;;
ops/runner/test_read_public_gitea_actions_queue.py)
;;
ops/runner/test_cd_controlled_runtime_profile.py)
;;
ops/runner/test_check_awoooi_non110_runner_readiness.py)
;;
ops/runner/test_install_awoooi_non110_runner_user_service.py)
;;
ops/runner/test_register_awoooi_110_controlled_cd_lane_drain.py)
;;
ops/runner/test_check_awoooi_110_controlled_cd_lane_readiness.py)
;;
ops/runner/test_verify_awoooi_non110_cd_closure.py)
;;
ops/runner/awoooi-cd-lane-drain.service)
;;
ops/runner/check-awoooi-110-controlled-cd-lane-readiness.sh)
;;
ops/runner/verify-awoooi-non110-cd-closure.py)
;;
docs/operations/host-cpu-pressure-drain-readback-2026-07-01.snapshot.json)
;;
docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json)
;;
ops/reboot-recovery/full-stack-cold-start-baseline.yml)
;;
ops/monitoring/alerts-unified.yml)
;;
ops/monitoring/alerts.yml)
;;
ops/alertmanager/alertmanager.yml)
;;
ops/reboot-recovery/full-stack-backup-baseline.yml)
;;
scripts/dev/awoooi-navigation-coverage-guard.py)
;;
scripts/ci/wait-host-web-build-pressure.sh)
;;
# 2026-07-01 Codex: backup freshness/readback scripts are
# covered by shell syntax checks and focused exporter tests.
# Keep them off B5 so a metadata/script recovery patch does not
# require the host Docker socket.
scripts/backup/backup-awoooi-frequent.sh)
;;
scripts/backup/backup-status.sh)
;;
scripts/backup/gitea-repo-bundle-backup.sh)
;;
scripts/backup/gitea-bundle-sample-restore-dry-run.sh)
;;
scripts/backup/tests/test_backup_status_contract.py)
;;
scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py)
;;
scripts/ops/backup-alert-label-contract-check.py)
;;
scripts/ops/backup-alert-live-visibility-check.py)
;;
scripts/ops/backup-from-110.sh)
;;
scripts/ops/backup-health-textfile-exporter.py)
;;
scripts/ops/docker-disk-pressure-retention-cleanup.py)
;;
scripts/ops/docker-health-monitor.sh)
;;
scripts/ops/dr-drill.sh)
;;
scripts/ops/gitea-queue-hook-backlog-playbook.py)
;;
scripts/ops/host-runaway-process-exporter.py)
;;
scripts/ops/host-sustained-load-controller.py)
;;
scripts/ops/host-sustained-load-evidence.py)
;;
scripts/ops/deploy-alertmanager-config.sh)
;;
scripts/ops/notify-awoooi-ops.sh)
;;
scripts/ops/pg-backup.sh)
;;
scripts/ops/tests/test_backup_health_textfile_exporter.py)
;;
scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py)
;;
scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py)
;;
scripts/ops/tests/test_host_runaway_process_exporter.py)
;;
scripts/ops/tests/test_host_pressure_alert_contract.py)
;;
scripts/ops/tests/test_alertmanager_webhook_config.py)
;;
scripts/reboot-recovery/deploy-to-110.sh)
;;
scripts/reboot-recovery/deploy-to-188.sh)
;;
scripts/reboot-recovery/enforce-110-runner-failclosed.sh)
;;
scripts/reboot-recovery/recover-110-control-path-and-harbor-local.sh)
;;
scripts/reboot-recovery/apply-credential-escrow-closeout-receipt-to-110.sh)
;;
scripts/reboot-recovery/awoooi-startup.sh)
;;
scripts/reboot-recovery/awoooi-startup.service)
;;
scripts/reboot-recovery/dr-escrow-evidence-checklist.py)
;;
scripts/reboot-recovery/apply-credential-escrow-closeout-receipt-to-110.sh)
;;
scripts/reboot-recovery/tests/test_dr_escrow_evidence_checklist.py)
;;
scripts/reboot-recovery/post-reboot-owner-response-preflight.py)
;;
scripts/reboot-recovery/post-start-quick-check.sh)
;;
scripts/reboot-recovery/reboot-recovery-readiness-audit.sh)
;;
scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh)
;;
scripts/reboot-recovery/full-stack-cold-start-check.sh)
;;
scripts/reboot-recovery/cold-start-textfile-exporter.sh)
;;
scripts/reboot-recovery/install-cold-start-monitor-110.sh)
;;
scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh)
;;
scripts/reboot-recovery/momo-source-arrival-gate.py)
;;
scripts/reboot-recovery/full-stack-recovery-scorecard.sh)
;;
scripts/reboot-recovery/harbor-watchdog.sh)
;;
scripts/reboot-recovery/awoooi-startup-110.sh)
;;
scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh)
;;
scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh)
;;
scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh)
;;
scripts/reboot-recovery/tests/test_dr_escrow_evidence_checklist.py)
;;
scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py)
;;
scripts/reboot-recovery/tests/test_momo_source_arrival_gate.py)
;;
scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service)
;;
scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer)
;;
scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh)
;;
scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh)
;;
scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh)
;;
scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py)
;;
scripts/reboot-recovery/collect-windows99-vmware-verify.sh)
;;
scripts/reboot-recovery/validate-windows99-console-verify-artifact.py)
;;
scripts/reboot-recovery/windows99-vmware-autostart.ps1)
;;
scripts/reboot-recovery/windows99-management-channel-probe.py)
;;
scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py)
;;
scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py)
;;
scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py)
;;
scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py)
;;
scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py)
;;
scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py)
;;
scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py)
;;
scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py)
;;
scripts/reboot-recovery/tests/test_recover_110_control_path_and_harbor_local.py)
;;
scripts/security/gitea-private-inventory-p0-scorecard.py)
;;
scripts/security/gitea-authenticated-inventory-payload-validator.py)
;;
scripts/security/security-mirror-progress-guard.py)
;;
scripts/security/telegram-notification-egress-no-new-bypass-guard.py)
;;
scripts/security/telegram-notification-egress-owner-response-acceptance.py)
;;
scripts/security/tests/test_gitea_private_inventory_p0_scorecard.py)
;;
scripts/security/tests/test_gitea_authenticated_inventory_payload_validator.py)
;;
*)
CONTROLLED_RUNTIME_TEST_PROFILE=0
;;
esac
done <> "$GITHUB_ENV"
echo "✅ controlled-runtime API test profile selected"
else
export AWOOOI_CD_TEST_PROFILE=full
echo "AWOOOI_CD_TEST_PROFILE=full" >> "$GITHUB_ENV"
echo "✅ full API test profile selected"
fi
printf '%s\n' "$AWOOOI_CD_TEST_PROFILE" > .awoooi-cd-test-profile
cat > /tmp/awoooi-api-tests.sh <<'CI_SCRIPT'
VENV=/opt/api-venv
HASH_FILE=/opt/api-venv/.deps_hash
CURRENT_HASH=$(md5sum apps/api/pyproject.toml | awk '{print $1}')
# python3.11 是 runner 層級持久安裝，只在首次或版本消失時才 apt-get
# 2026-04-05 Claude Code: 分離 apt-get 與 venv hash-guard，避免每次 deps 變更都重跑 apt
# 2026-04-16 ogt + Claude Sonnet 4.6: 修復 apt index 失敗 → 改用 --fix-missing + retry
if ! command -v python3.11 &>/dev/null; then
echo "📦 安裝 python3.11..."
apt-get clean && rm -rf /var/lib/apt/lists/*
apt-get update -q --fix-missing || apt-get update -q || true
apt-get install -y -q python3.11-venv python3.11 || \
(add-apt-repository ppa:deadsnakes/python -y 2>/dev/null && apt-get update -q && apt-get install -y -q python3.11-venv python3.11) || true
else
echo "⚡ python3.11 已安裝，跳過 apt-get"
fi
# 確保 python3.11 存在，否則 fallback 到系統 python3
if ! command -v python3.11 &>/dev/null; then
echo "⚠️ python3.11 安裝失敗，使用 python3 fallback"
ln -sf "$(which python3)" /usr/local/bin/python3.11 || true
fi
if [ ! -d "$VENV/bin" ] || [ "$(cat $HASH_FILE 2>/dev/null)" != "$CURRENT_HASH" ]; then
echo "📦 deps 已變更，重建 venv..."
# 2026-04-17 ogt: /opt/api-venv 是 volume mount，不能 rm -rf 目錄本身
# 改用 find 清空內容，保留 mount point 目錄
find "$VENV" -mindepth 1 -delete 2>/dev/null || true
python3.11 -m venv $VENV
source $VENV/bin/activate
pip install -q uv
cd apps/api && uv pip install -q -e ".[dev]" && cd -
echo "$CURRENT_HASH" > $HASH_FILE
else
echo "⚡ 使用快取 venv (deps 未變更)"
source $VENV/bin/activate
fi
cd apps/api
cleanup_pytest_workspace_cache() {
# 2026-05-19 Codex: CI image runs as root against a bind-mounted
# checkout. Remove Python cache artifacts before act-runner cleanup
# so successful jobs do not end with root-owned __pycache__ noise.
find tests src -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true
rm -rf .pytest_cache 2>/dev/null || true
}
# CI 排除需外部服務的測試 (Redis pool / Ollama — 2026-04-01 Claude Code)
# 2026-04-05 Claude Code: 修正 exit code — | tail 會吃掉 segfault (exit 139)
# 改用 tee + PIPESTATUS[0] 正確捕捉 pytest 本身的 exit code
# 2026-04-05 Claude Code: 加 --ignore=tests/integration 排除需 asyncpg 連線的 DB 測試
# integration tests 在 prod K8s 部署後由 E2E Smoke Test 覆蓋
# PYTHONFAULTHANDLER=1: 若 C extension segfault，輸出完整 Python stacktrace
# 2026-04-05 Claude Code: test_github_webhook.py 已根治
# 原問題: import src.main → asyncpg C ext segfault (exit 139)
# 修復: 改用最小化 app，只掛載 github_webhook router，不走 DB import chain
# 現在可安全加入 CI 測試
# 2026-04-22 ogt: DATABASE_URL 改為必填後，單元測試需要此 env var 讓 Settings 通過驗證
# 單元測試不連 DB，此 CI placeholder 僅供 Pydantic 驗證，不產生真實連線
if [ "${AWOOOI_CD_TEST_PROFILE:-full}" = "controlled-runtime" ]; then
echo "✅ controlled-runtime profile: running focused replay/auto-approve/copy tests"
python3.11 ../../scripts/ci/generate-cicd-artifact-receipts.py --repo-root ../.. --out-dir /tmp/awoooi-cicd-artifacts
echo "package-release-promotion source_contract=awoooi-ci-package-release-promotion.json"
python3.11 -m py_compile \
src/core/config.py \
src/db/base.py \
src/api/v1/platform/events.py \
src/api/v1/agents.py \
src/api/v1/iwooos.py \
src/api/v1/webhooks.py \
src/jobs/ai_slo_watchdog_job.py \
src/repositories/knowledge_repository.py \
src/models/knowledge.py \
src/models/playbook.py \
src/services/knowledge_service.py \
src/services/awoooi_production_deploy_readback_blocker.py \
src/services/agent_replay_normalizer.py \
src/services/ai_agent_log_intelligence_integration_readback.py \
src/services/ai_agent_log_feedback_receipt_dry_run.py \
src/services/ai_agent_log_post_write_verifier_dry_run.py \
src/services/ai_agent_log_controlled_writeback_plan_readback.py \
src/services/ai_agent_log_controlled_writeback_executor_readback.py \
src/services/ai_agent_log_controlled_writeback_dispatch.py \
src/services/ai_agent_log_controlled_writeback_consumer_readback.py \
src/services/ai_agent_log_controlled_writeback_consumer_apply.py \
src/services/ai_agent_autonomous_runtime_control.py \
src/services/executor_trust_boundary_readback.py \
src/services/awooop_ansible_audit_service.py \
src/services/awooop_ansible_check_mode_service.py \
src/workers/ansible_executor_broker.py \
src/workers/signal_worker.py \
src/services/auto_repair_service.py \
src/services/auto_approve.py \
src/services/backup_dr_target_inventory.py \
src/services/backup_dr_readiness_matrix.py \
src/services/gitea_repo_bundle_backup_readback.py \
src/services/decision_fusion.py \
src/services/delivery_closure_workbench.py \
src/services/heartbeat_report_service.py \
src/services/credential_escrow_evidence_intake_readiness.py \
src/services/gitea_authenticated_inventory_payload_validation.py \
src/services/gitea_owner_coverage_attestation_validation.py \
src/services/gitea_private_inventory_closeout_validation.py \
src/services/gitea_private_inventory_p0_scorecard.py \
src/services/gitea_workflow_runner_owner_attestation_request.py \
src/services/gitea_cicd_alert_receipt_chain_readback.py \
src/services/gitea_capability_ci_cd_optimization_readback.py \
src/services/reboot_auto_recovery_slo_scorecard.py \
src/services/reboot_auto_recovery_drill_preflight.py \
src/services/stockplatform_public_api_runtime_readback.py \
src/services/stockplatform_public_api_controlled_recovery_preflight.py \
src/services/harbor_registry_controlled_recovery_preflight.py \
src/services/harbor_registry_controlled_recovery_receipt.py \
src/services/iwooos_security_operating_system.py \
src/services/awoooi_gitea_onboarding_warning_step_dashboard.py \
src/services/awoooi_gitea_onboarding_warning_step_owner_package.py \
src/services/awoooi_gitea_onboarding_warning_step_owner_response_preflight.py \
src/services/awoooi_gitea_onboarding_warning_step_template_copy_apply_gate.py \
src/services/awoooi_gitea_onboarding_warning_step_template_copy_execution_plan.py \
src/services/awoooi_gitea_onboarding_warning_step_template_copy_receipt.py \
src/services/awoooi_gitea_onboarding_warning_step_runtime_enablement_gate.py \
src/services/awoooi_new_product_onboarding_page_model.py \
src/services/awoooi_onboarding_reminder_contract.py \
src/services/awoooi_onboarding_source_contracts.py \
src/services/awoooi_priority_work_order_readback.py \
src/services/awoooi_product_onboarding_guard.py \
src/api/v1/platform/operator_runs.py \
src/services/channel_hub.py \
src/services/p0_cicd_baseline_source_readiness.py \
src/services/product_awoooi_manifest_standard.py \
src/services/platform_operator_service.py \
src/services/telegram_alert_ai_automation_matrix.py \
src/services/telegram_alert_monitoring_coverage_readback.py \
src/services/telegram_gateway.py
python3.11 -m py_compile \
../../scripts/reboot-recovery/dr-escrow-evidence-checklist.py \
../../scripts/reboot-recovery/post-reboot-owner-response-preflight.py \
../../scripts/reboot-recovery/momo-source-arrival-gate.py \
../../scripts/reboot-recovery/public-maintenance-fallback-probe.py \
../../scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py \
../../scripts/reboot-recovery/validate-windows99-console-verify-artifact.py \
../../scripts/reboot-recovery/windows99-management-channel-probe.py \
../../scripts/ops/backup-alert-label-contract-check.py \
../../scripts/ops/backup-alert-live-visibility-check.py \
../../scripts/ops/backup-health-textfile-exporter.py \
../../scripts/ops/docker-disk-pressure-retention-cleanup.py \
../../scripts/ops/gitea-queue-hook-backlog-playbook.py \
../../scripts/ops/host-runaway-process-exporter.py \
../../scripts/ops/host-sustained-load-controller.py \
../../scripts/ops/host-sustained-load-evidence.py \
../../scripts/security/gitea-private-inventory-p0-scorecard.py \
../../scripts/security/gitea-authenticated-inventory-payload-validator.py \
../../scripts/security/runtime_image_mirror_controller.py \
../../scripts/security/security-mirror-progress-guard.py \
../../scripts/security/telegram-notification-egress-no-new-bypass-guard.py \
../../scripts/security/telegram-notification-egress-owner-response-acceptance.py
python3.11 -m pytest ../../scripts/ci/tests/test_generate_cicd_artifact_receipts.py -q --tb=short -p no:cacheprovider
python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts-unified.yml')); print('alerts-unified YAML OK')"
python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts.yml')); print('alerts YAML OK')"
python3.11 -c "import yaml; yaml.safe_load(open('../../ops/alertmanager/alertmanager.yml')); print('alertmanager YAML OK')"
python3.11 -c "import yaml; yaml.safe_load(open('../../ops/reboot-recovery/full-stack-cold-start-baseline.yml')); print('full-stack-cold-start-baseline YAML OK')"
python3.11 -c "import yaml; yaml.safe_load(open('../../ops/reboot-recovery/full-stack-backup-baseline.yml')); print('full-stack-backup-baseline YAML OK')"
bash -n \
../../ops/runner/check-awoooi-110-controlled-cd-lane-readiness.sh \
../../ops/runner/check-awoooi-non110-runner-readiness.sh \
../../ops/runner/install-awoooi-non110-runner-user-service.sh \
../../ops/runner/register-awoooi-110-controlled-cd-lane-drain.sh \
../../scripts/reboot-recovery/deploy-to-110.sh \
../../scripts/reboot-recovery/deploy-to-188.sh \
../../scripts/reboot-recovery/enforce-110-runner-failclosed.sh \
../../scripts/reboot-recovery/recover-110-control-path-and-harbor-local.sh \
../../scripts/reboot-recovery/awoooi-startup.sh \
../../scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh \
../../scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh \
../../scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh \
../../scripts/reboot-recovery/collect-windows99-vmware-verify.sh \
../../scripts/reboot-recovery/post-start-quick-check.sh \
../../scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh \
../../scripts/reboot-recovery/full-stack-cold-start-check.sh \
../../scripts/reboot-recovery/cold-start-textfile-exporter.sh \
../../scripts/reboot-recovery/install-cold-start-monitor-110.sh \
../../scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh \
../../scripts/reboot-recovery/full-stack-recovery-scorecard.sh \
../../scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh \
../../scripts/reboot-recovery/harbor-watchdog.sh \
../../scripts/reboot-recovery/awoooi-startup-110.sh \
../../scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh \
../../scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh \
../../scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh \
../../scripts/reboot-recovery/apply-credential-escrow-closeout-receipt-to-110.sh \
../../scripts/backup/backup-awoooi-frequent.sh \
../../scripts/backup/backup-status.sh \
../../scripts/backup/gitea-repo-bundle-backup.sh \
../../scripts/backup/gitea-bundle-sample-restore-dry-run.sh \
../../scripts/ops/backup-from-110.sh \
../../scripts/ops/deploy-alertmanager-config.sh \
../../scripts/ops/docker-health-monitor.sh \
../../scripts/ops/dr-drill.sh \
../../scripts/ops/notify-awoooi-ops.sh \
../../scripts/ops/pg-backup.sh
bash -n ../../scripts/reboot-recovery/apply-credential-escrow-closeout-receipt-to-110.sh
DATABASE_URL="${DATABASE_URL:-postgresql+asyncpg://ci:ci@localhost/ci}" \
PYTHONFAULTHANDLER=1 python3.11 -m pytest \
tests/test_agent_replay_normalizer.py \
tests/test_ai_agent_log_intelligence_integration_readback_api.py \
tests/test_ai_agent_log_feedback_receipt_dry_run_api.py \
tests/test_ai_agent_log_post_write_verifier_dry_run_api.py \
tests/test_ai_agent_log_controlled_writeback_plan_readback_api.py \
tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py \
tests/test_ai_agent_log_controlled_writeback_dispatch_api.py \
tests/test_ai_agent_log_controlled_writeback_consumer_readback_api.py \
tests/test_ai_agent_log_controlled_writeback_consumer_apply_api.py \
tests/test_ai_agent_autonomous_runtime_control.py \
tests/test_executor_trust_boundary_readback.py \
tests/test_ai_agent_report_truth_actionability_review.py \
tests/test_ai_agent_report_truth_actionability_review_api.py \
tests/test_awooop_truth_chain_service.py \
tests/test_signal_worker_ansible_executor_binding.py \
tests/test_channel_hub_grouped_alert_events.py \
tests/test_shadow_auto_approve.py \
tests/test_destructive_patterns.py \
tests/test_approval_pending_visibility.py \
tests/test_awooop_operator_timeline_labels.py::test_outbound_timeline_title_labels_runbook_review \
tests/test_config_url_validation.py \
tests/test_delivery_closure_workbench_api.py \
tests/test_runtime_bootstrap_guards.py \
tests/test_backup_dr_target_inventory.py \
tests/test_backup_dr_target_inventory_api.py \
tests/test_backup_dr_readiness_matrix.py \
tests/test_backup_dr_readiness_matrix_api.py \
tests/test_gitea_repo_bundle_backup_readback_api.py \
tests/test_gitea_cicd_alert_receipt_chain_readback_api.py \
tests/test_gitea_capability_ci_cd_optimization_readback_api.py \
tests/test_credential_escrow_evidence_intake_readiness_api.py \
tests/test_gitea_private_inventory_p0_scorecard_api.py \
tests/test_gitea_workflow_runner_owner_attestation_request_api.py \
tests/test_reboot_auto_recovery_slo_scorecard_api.py \
tests/test_windows99_vmx_source_locator_readback_api.py \
tests/test_stockplatform_public_api_runtime_readback.py \
tests/test_stockplatform_public_api_controlled_recovery_preflight.py \
tests/test_harbor_registry_controlled_recovery_preflight.py \
tests/test_harbor_registry_controlled_recovery_receipt.py \
tests/test_iwooos_security_operating_system.py \
tests/test_awoooi_production_deploy_readback_blocker.py \
tests/test_awoooi_priority_work_order_readback_api.py \
tests/test_alertmanager_webhook_metrics.py \
tests/test_telegram_alert_ai_automation_matrix_api.py \
tests/test_telegram_alert_monitoring_coverage_readback_api.py \
tests/e2e_network_test.py::TestHMACVerification::test_valid_hmac_signature \
tests/test_p0_cicd_baseline_source_readiness_api.py \
tests/test_product_awoooi_manifest_standard_api.py \
tests/test_trust_drift_watchdog.py \
../../ops/runner/test_read_public_gitea_actions_queue.py \
../../ops/runner/test_cd_controlled_runtime_profile.py \
../../ops/runner/test_check_awoooi_non110_runner_readiness.py \
../../ops/runner/test_install_awoooi_non110_runner_user_service.py \
../../ops/runner/test_register_awoooi_110_controlled_cd_lane_drain.py \
../../ops/runner/test_check_awoooi_110_controlled_cd_lane_readiness.py \
../../ops/runner/test_verify_awoooi_non110_cd_closure.py \
../../scripts/backup/tests/test_backup_status_contract.py \
../../scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py \
../../scripts/ops/tests/test_backup_health_textfile_exporter.py \
../../scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py \
../../scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py \
../../scripts/ops/tests/test_host_runaway_process_exporter.py \
../../scripts/ops/tests/test_host_pressure_alert_contract.py \
../../scripts/ops/tests/test_alertmanager_webhook_config.py \
../../scripts/reboot-recovery/tests/test_dr_escrow_evidence_checklist.py \
../../scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py \
../../scripts/reboot-recovery/tests/test_momo_source_arrival_gate.py \
../../scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py \
../../scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py \
../../scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py \
../../scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py \
../../scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py \
../../scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py \
../../scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py \
../../scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py \
../../scripts/reboot-recovery/tests/test_recover_110_control_path_and_harbor_local.py \
../../scripts/security/tests/test_gitea_private_inventory_p0_scorecard.py \
../../scripts/security/tests/test_gitea_authenticated_inventory_payload_validator.py \
../../scripts/security/tests/test_runtime_image_mirror_controller.py \
-v --tb=short -x -p no:cacheprovider \
2>&1 | tee /tmp/pytest-output.txt; PYTEST_EXIT=${PIPESTATUS[0]}
else
python3.11 ../../scripts/ci/generate-cicd-artifact-receipts.py --repo-root ../.. --out-dir /tmp/awoooi-cicd-artifacts
echo "package-release-promotion source_contract=awoooi-ci-package-release-promotion.json"
echo "✅ full profile: running learning-chain E2E in isolated pytest process"
DATABASE_URL="${DATABASE_URL:-postgresql+asyncpg://ci:ci@localhost/ci}" \
PYTHONFAULTHANDLER=1 python3.11 -m pytest tests/test_learning_chain_e2e.py -v --tb=short -p no:cacheprovider \
2>&1 | tee /tmp/pytest-learning-chain-output.txt; LEARNING_CHAIN_EXIT=${PIPESTATUS[0]}
cat /tmp/pytest-learning-chain-output.txt > /tmp/pytest-output.txt
if [ "$LEARNING_CHAIN_EXIT" -ne 0 ]; then
tail -60 /tmp/pytest-output.txt
cleanup_pytest_workspace_cache
exit "$LEARNING_CHAIN_EXIT"
fi
DATABASE_URL="${DATABASE_URL:-postgresql+asyncpg://ci:ci@localhost/ci}" \
PYTHONFAULTHANDLER=1 python3.11 -m pytest tests/ -v --tb=short -x -p no:cacheprovider \
--ignore=tests/integration \
--ignore=tests/test_anomaly_counter.py \
--ignore=tests/test_global_repair_cooldown.py \
--ignore=tests/test_redis_multisig.py \
--ignore=tests/test_model_regression.py \
--ignore=tests/test_prompt_validation.py \
--ignore=tests/test_learning_chain_e2e.py \
--ignore=tests/e2e_network_test.py \
2>&1 | tee -a /tmp/pytest-output.txt; PYTEST_EXIT=${PIPESTATUS[0]}
fi
tail -60 /tmp/pytest-output.txt
cleanup_pytest_workspace_cache
exit $PYTEST_EXIT
CI_SCRIPT
docker run --rm \
--name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-api-tests" \
-e AWOOOI_CD_TEST_PROFILE="${AWOOOI_CD_TEST_PROFILE:-full}" \
--cpus "2.0" \
--memory "6g" \
--memory-swap "8g" \
-v "$PWD:/workspace" \
-v /tmp/awoooi-api-tests.sh:/tmp/awoooi-api-tests.sh:ro \
-v awoooi-api-venv-cache:/opt/api-venv \
-w /workspace \
"${{ env.CI_IMAGE }}" \
bash /tmp/awoooi-api-tests.sh
# ── 整合測試 B5 (2026-04-10) ──────────────────────────────────────────
# B5 整合測試 — postgres-test 由 services: 提供，localhost:15432 直連
# 2026-04-10 Claude Sonnet 4.6: 用 psql 直連 localhost:15432 初始化 schema
# (docker exec 在 act runner 內無法取得 service container name)
# B5: Gitea act runner 的 services: 實作與 GitHub Actions 不同
# service container 啟動後需直連，但 act 的 container name 可能為空
# 2026-04-10 ogt: 改用 docker run 本地啟動取代 services: 宣告
# 2026-04-19 ogt + Claude Opus 4.7: cd 連續 2 次 fail (run 984/985)
# 真因: act runner 把 ci-runner 跑在獨立 user-defined network,
# pg-test-b5 預設用 host bridge → 兩邊隔離無法連 (172.17.0.2 timeout)
# 修法: 把 pg-test-b5 加入 act task 的 network,用 container name 連線
- name: Integration Tests (B5 — 真實 DB)
run: |
if [ -z "${AWOOOI_CD_TEST_PROFILE:-}" ] && [ -f .awoooi-cd-test-profile ]; then
AWOOOI_CD_TEST_PROFILE="$(tr -d '\r\n' < .awoooi-cd-test-profile)"
export AWOOOI_CD_TEST_PROFILE
fi
echo "B5 effective test profile=${AWOOOI_CD_TEST_PROFILE:-full}"
if [ "${AWOOOI_CD_TEST_PROFILE:-full}" = "controlled-runtime" ]; then
echo "✅ controlled-runtime profile: B5 DB integration unchanged; skipping B5 for this narrow release lane"
exit 0
fi
if ! docker info >/dev/null 2>&1; then
echo "BLOCKER b5_outer_docker_unavailable"
echo "NEXT_ACTION ensure_cd_runner_host_can_start_pgvector_container_then_retry_cd"
exit 65
fi
B5_RUN_ID_RAW="${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}"
B5_RUN_ID_SAFE="$(printf '%s' "$B5_RUN_ID_RAW" | tr -c 'A-Za-z0-9_.-' '-')"
B5_NET="b5-test-net-${B5_RUN_ID_SAFE}"
B5_DB_CONTAINER="pg-test-b5-${B5_RUN_ID_SAFE}"
cleanup_b5() {
docker rm -f "$B5_DB_CONTAINER" >/dev/null 2>&1 || true
docker network rm "$B5_NET" >/dev/null 2>&1 || true
}
trap cleanup_b5 EXIT
cleanup_b5
docker network create "$B5_NET" >/dev/null
docker run -d --name "$B5_DB_CONTAINER" \
--network="$B5_NET" \
--network-alias pg-test-b5 \
-e POSTGRES_DB=awoooi_test \
-e POSTGRES_USER=awoooi \
-e POSTGRES_PASSWORD=awoooi_test_2026 \
pgvector/pgvector:pg16
B5_DB_READY=0
for i in $(seq 1 30); do
if docker exec "$B5_DB_CONTAINER" pg_isready -U awoooi -d awoooi_test; then
B5_DB_READY=1
break
fi
sleep 2
done
if [ "$B5_DB_READY" != "1" ]; then
echo "BLOCKER b5_pg_test_container_not_ready"
echo "NEXT_ACTION inspect_b5_test_network_and_outer_docker_then_retry_cd"
docker ps --filter "name=$B5_DB_CONTAINER" --format 'b5_container={{.Names}} status={{.Status}}' || true
exit 66
fi
cat > /tmp/awoooi-b5-tests.sh <<'CI_SCRIPT'
set -euo pipefail
cd apps/api
# 安裝 psql client
if ! command -v psql &>/dev/null; then
apt-get install -y -q postgresql-client
fi
B5_DB_HOST="${B5_DB_HOST:-pg-test-b5}"
# 初始化 schema
PGPASSWORD=awoooi_test_2026 psql \
-h "$B5_DB_HOST" -p 5432 -U awoooi -d awoooi_test \
-f tests/integration/setup_test_schema.sql
# 跑測試
# B5 整合測試嚴格模式 (2026-04-13 ogt: 恢復 Break-Glass 移除)
# -m integration: override pyproject.toml addopts "-m 'not integration'"，讓標記測試可執行
# 2026-04-22 ogt: DATABASE_URL 改為必填後，import chain 需要此 env var 讓 Settings 通過驗證
DATABASE_URL="postgresql+asyncpg://awoooi:awoooi_test_2026@${B5_DB_HOST}:5432/awoooi_test?ssl=disable" \
TEST_DATABASE_URL="postgresql+asyncpg://awoooi:awoooi_test_2026@${B5_DB_HOST}:5432/awoooi_test?ssl=disable" \
/opt/api-venv/bin/pytest tests/integration/test_b5_core_flows.py -v --tb=short -m integration -p no:cacheprovider || PYTEST_EXIT=$?
# pg-test-b5 is cleaned by the outer runner trap after this container exits.
# 2026-05-20 Codex: B5 imports shared tests helpers, so cleanup the
# whole tests tree to avoid root-owned __pycache__ act-runner noise.
find tests src -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true
rm -rf .pytest_cache 2>/dev/null || true
exit "${PYTEST_EXIT:-0}"
CI_SCRIPT
docker run --rm \
--name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-b5-tests" \
--user 0:0 \
--cpus "2.0" \
--memory "2g" \
--network "$B5_NET" \
-e B5_DB_HOST=pg-test-b5 \
-v "$PWD:/workspace" \
-v /tmp/awoooi-b5-tests.sh:/tmp/awoooi-b5-tests.sh:ro \
-v awoooi-api-venv-cache:/opt/api-venv \
-w /workspace \
"${{ env.CI_IMAGE }}" \
bash /tmp/awoooi-b5-tests.sh
- name: Clean Test Workspace Artifacts
if: always()
env:
HOST_RUNNER_CLEANUP_IMAGE: ${{ env.CI_IMAGE }}
run: bash scripts/ci/cleanup-host-runner-workspace.sh
- name: Notify Pipeline Failure
# 2026-04-30 Codex: tests job failure notifier; no jq dependency for host parity.
if: failure()
env:
TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
run: |
COMMIT_MSG="${{ steps.commit.outputs.message }}"
SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
ACTOR="${{ github.actor }}"
COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&/g; s//\>/g')
MSG=$(printf '❌ AWOOOI 部署失敗\n├ 📝 %s\n├ 🔖 %s\n├ 👤 %s\n├ 🧪 Stage: tests\n└ 🔗 http://192.168.0.110:3001/wooo/awoooi/actions' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}")
if AWOOI_CICD_STATUS=failed \
AWOOI_CICD_STAGE=tests \
AWOOI_CICD_JOB_NAME="AWOOOI 部署失敗" \
AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \
scripts/ci/notify-awoooi-cicd.sh; then
echo "✅ CI/CD tests failure notification mirrored through AWOOI API"
else
echo "AWOOI API notify failed; direct Telegram fallback disabled to preserve AwoooP receipt chain"
fi
build-and-deploy:
# 2026-06-28 Codex: keep CD-generated `[skip ci]` deploy commits and
# `cancel-stale-cd` queue-cleaning commits from re-entering build/deploy.
# 2026-07-01 Codex: metadata-only controlled-runtime fixes already run the
# focused tests above; do not spend the Docker build lock or deploy marker
# when no app image can change.
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]')) }}
# 2026-04-30 Codex: Docker builds run on the host runner. Long docker build
# steps were killing the transient act job container with RWLayer=nil.
needs: [tests]
timeout-minutes: 60
runs-on: awoooi-non110-host
steps:
- name: Bootstrap Host Runner Tools
# 2026-05-05 Codex: keep the host-mode runner self-healing before
# actions/checkout@v4 and Telegram failure notifications run.
run: |
if command -v apk >/dev/null 2>&1; then
apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
fi
- name: Checkout source from Gitea
env:
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
run: |
set -euo pipefail
git init .
git remote remove origin 2>/dev/null || true
git remote add origin "$GITEA_SOURCE_URL"
git fetch --no-tags --prune --depth=1 origin "$GITHUB_SHA"
# Deploy marker creation and push require a real local main ref.
git checkout --force -B main FETCH_HEAD
- name: Wait for Host Web Build Pressure
# 2026-06-27 Codex: post-deploy smoke is also browser-heavy. Refuse to
# add another smoke run while active CI/build/smoke pressure is present.
run: bash scripts/ci/wait-host-web-build-pressure.sh
- name: Get Commit Info
id: commit
run: |
echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
echo "message=$(git log -1 --pretty=%s | head -c 50)" >> $GITHUB_OUTPUT
echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT
- name: Notify Build Deploy Start
run: |
ACTOR="${{ github.actor }}"
if AWOOI_CICD_STATUS=running \
AWOOI_CICD_STAGE=build-and-deploy \
AWOOI_CICD_JOB_NAME="AWOOOI 建置部署開始" \
AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
AWOOI_CICD_SUMMARY="${{ steps.commit.outputs.message }}" \
scripts/ci/notify-awoooi-cicd.sh; then
echo "✅ CI/CD build-deploy start notification mirrored through AWOOI API"
else
echo "⚠️ CI/CD build-deploy start notification failed (non-fatal)"
fi
- name: Login to Harbor
env:
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
run: |
HARBOR_REGISTRY="${{ env.HARBOR }}"
LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-3}"
LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-5}"
WATCHDOG="/usr/local/bin/harbor-watchdog.sh"
HARBOR_110_NODE_EXPORTER_URL="${HARBOR_110_NODE_EXPORTER_URL:-http://192.168.0.110:9100/metrics}"
HARBOR_110_MAX_LOAD5_PER_CORE="${HARBOR_110_MAX_LOAD5_PER_CORE:-1.25}"
HARBOR_110_MAX_GITEA_CPU_CORES="${HARBOR_110_MAX_GITEA_CPU_CORES:-3.0}"
probe_registry_v2() {
curl --silent --show-error \
--output /dev/null \
--write-out "%{http_code}" \
--max-time 10 \
"https://${HARBOR_REGISTRY}/v2/" || true
}
host_has_110_ip() {
command -v ip >/dev/null 2>&1 || return 1
ip -o -4 addr show 2>/dev/null | grep -q " 192.168.0.110/"
}
run_watchdog() {
if [ "$(id -u)" = "0" ]; then
"${WATCHDOG}" "$@"
else
sudo -n "${WATCHDOG}" "$@"
fi
}
greater_than() {
awk -v left="$1" -v right="$2" 'BEGIN { exit !(left > right) }'
}
preflight_110_capacity() {
metrics="$(curl --silent --show-error --max-time 5 "${HARBOR_110_NODE_EXPORTER_URL}" || true)"
if [ -z "${metrics}" ]; then
echo "harbor_110_capacity_metrics_unavailable=1"
return 0
fi
load5_per_core="$(printf '%s\n' "${metrics}" | awk '$1 ~ /^awoooi_host_load5_per_core\\{/ && $0 ~ /host=\"110\"/ { value = $NF } END { print value }')"
gitea_cpu_cores="$(printf '%s\n' "${metrics}" | awk '$1 ~ /^docker_container_cpu_cores\\{/ && $0 ~ /container_name=\"gitea\"/ { value = $NF } END { print value }')"
if [ -n "${load5_per_core}" ]; then
echo "harbor_110_load5_per_core=${load5_per_core}"
if greater_than "${load5_per_core}" "${HARBOR_110_MAX_LOAD5_PER_CORE}"; then
echo "BLOCKER harbor_110_host_pressure_high load5_per_core=${load5_per_core} threshold=${HARBOR_110_MAX_LOAD5_PER_CORE}"
return 1
fi
fi
if [ -n "${gitea_cpu_cores}" ]; then
echo "harbor_110_gitea_cpu_cores=${gitea_cpu_cores}"
if greater_than "${gitea_cpu_cores}" "${HARBOR_110_MAX_GITEA_CPU_CORES}"; then
echo "BLOCKER harbor_110_gitea_cpu_saturated cpu_cores=${gitea_cpu_cores} threshold=${HARBOR_110_MAX_GITEA_CPU_CORES}"
return 1
fi
fi
return 0
}
controlled_harbor_repair_once() {
if [ "${AWOOOI_CD_HARBOR_CONTROLLED_REPAIR:-1}" != "1" ]; then
echo "harbor_controlled_repair_skipped=disabled"
return 1
fi
if ! host_has_110_ip; then
echo "harbor_controlled_repair_skipped=not_110_host"
return 1
fi
if [ ! -x "${WATCHDOG}" ]; then
echo "harbor_controlled_repair_skipped=watchdog_missing"
return 1
fi
echo "harbor_controlled_repair_check_start=1"
set +e
check_output="$(run_watchdog --check 2>&1)"
check_rc=$?
set -e
printf '%s\n' "${check_output}" | sed -n '1,80p'
echo "harbor_controlled_repair_check_rc=${check_rc}"
if [ "${check_rc}" -ne 0 ]; then
return 1
fi
if ! printf '%s\n' "${check_output}" | grep -q "AWOOOI_HARBOR_WATCHDOG_CHECK"; then
echo "harbor_controlled_repair_blocked=missing_watchdog_check_marker"
return 1
fi
if printf '%s\n' "${check_output}" | grep -q "harbor_ready=true"; then
echo "harbor_controlled_repair_skipped=already_ready"
return 0
fi
echo "harbor_controlled_repair_once_start=1"
set +e
repair_output="$(run_watchdog --repair-once 2>&1)"
repair_rc=$?
set -e
printf '%s\n' "${repair_output}" | sed -n '1,140p'
echo "harbor_controlled_repair_once_rc=${repair_rc}"
[ "${repair_rc}" -eq 0 ]
}
attempt=1
repair_attempted=0
if ! preflight_110_capacity; then
echo "NEXT_ACTION wait_for_110_load_to_normalize_then_rerun_harbor_watchdog_check"
echo "NEXT_ACTION if_110_load_stays_high_use_local_console: sudo /usr/local/bin/recover-110-control-path-and-harbor-local.sh --check"
exit 1
fi
while [ "${attempt}" -le "${LOGIN_ATTEMPTS}" ]; do
registry_status="$(probe_registry_v2)"
if [ -z "${registry_status}" ]; then
registry_status="000"
fi
if [ "${registry_status}" = "200" ] || [ "${registry_status}" = "401" ]; then
if printf '%s\n' "${HARBOR_PASSWORD}" | \
docker login "${HARBOR_REGISTRY}" \
-u "${HARBOR_USERNAME}" \
--password-stdin; then
echo "harbor_login_ready=1"
exit 0
fi
echo "harbor_login_attempt=${attempt} docker_login_failed"
else
echo "harbor_login_attempt=${attempt} registry_v2_status=${registry_status}"
if [ "${repair_attempted}" = "0" ]; then
repair_attempted=1
if controlled_harbor_repair_once; then
registry_status="$(probe_registry_v2)"
if [ -z "${registry_status}" ]; then
registry_status="000"
fi
echo "harbor_controlled_repair_public_registry_v2_status=${registry_status}"
continue
fi
fi
fi
if [ "${attempt}" -ge "${LOGIN_ATTEMPTS}" ]; then
echo "BLOCKER harbor_registry_public_route_unavailable registry_v2_status=${registry_status}"
echo "NEXT_ACTION run_on_110_local_console_or_restored_ssh: sudo /usr/local/bin/harbor-watchdog.sh --check"
echo "NEXT_ACTION if_check_confirms_unhealthy_on_110: sudo /usr/local/bin/harbor-watchdog.sh --repair-once"
echo "NEXT_ACTION combined_110_control_path_then_harbor: sudo /usr/local/bin/recover-110-control-path-and-harbor-local.sh --apply-all"
echo "NEXT_ACTION controlled_workflow_dispatch: .gitea/workflows/harbor-110-local-repair.yaml"
exit 1
fi
sleep "${LOGIN_SLEEP_SECONDS}"
attempt=$((attempt + 1))
done
# 2026-05-21 Codex: AWOOI workflow concurrency and the Docker network
# lock only protect AWOOI/Docker work. Other repos can still run
# host-side Next/Turbo builds on the same 110 runner and starve this
# deploy. Wait for those foreign web builds before starting our image
# build; the gate is read-only and never kills another process.
- name: Wait for Host Web Build Pressure
run: bash scripts/ci/wait-host-web-build-pressure.sh
# 2026-04-30 Codex: Gitea act-runner shares one Docker daemon across repos.
# When another repo starts a heavy docker build while AWOOOI Web is still
# building, the job container can disappear and Docker reports RWLayer=nil.
# A Docker-network lock is global to the host daemon and survives container
# namespaces, unlike /tmp/flock inside the transient job container.
# 2026-06-28 Codex: 110 runner pressure remains incident-grade; Docker
# build lock contention is fail-hard until non-110 readiness is verified.
- name: Acquire Docker Build Lock
run: |
LOCK_NAME="awoooi-cd-docker-build-lock"
LOCK_WARN_ONLY="${DOCKER_BUILD_LOCK_WARN_ONLY:-0}"
STALE_SECONDS="${DOCKER_BUILD_LOCK_STALE_SECONDS:-7200}"
EMPTY_LOCK_SECONDS="${DOCKER_BUILD_LOCK_EMPTY_SECONDS:-300}"
WAIT_ATTEMPTS="${DOCKER_BUILD_LOCK_WAIT_ATTEMPTS:-180}"
WAIT_SLEEP_SECONDS="${DOCKER_BUILD_LOCK_SLEEP_SECONDS:-10}"
for attempt in $(seq 1 "$WAIT_ATTEMPTS"); do
if docker network create \
--label awoooi.ci-lock=docker-build \
--label awoooi.owner=cd-pipeline \
"$LOCK_NAME" >/dev/null 2>&1; then
echo "DOCKER_BUILD_LOCK=${LOCK_NAME}" >> "$GITHUB_ENV"
echo "✅ Docker build lock acquired: ${LOCK_NAME}"
exit 0
fi
CREATED_AT=$(docker network inspect "$LOCK_NAME" \
--format '{{.Created}}' 2>/dev/null || true)
if [ -n "$CREATED_AT" ]; then
# 2026-05-03 ogt: 修復 stale 偵測 — Docker 回傳 "2006-01-02 15:04:05.999999999 -0700 MST"
# date -d 不接受奈秒小數點與末尾時區縮寫（CST/MST 等），導致 CREATED_EPOCH=0 → stale 永不觸發
# 2026-06-18 Codex: act-runner 容器可能沒有 GNU date / python3；
# node 由 bootstrap 安裝，作為 Docker CreatedAt 的穩定解析 fallback。
# 2026-06-19 Codex: Docker / Gitea runner 可能回傳 ISO
# `2026-06-18T16:20:00.123456789Z`；若 CREATED_EPOCH=0，
# empty lock 永遠不會自清，下一輪 deploy 會卡滿 30 分鐘。
CREATED_CLEAN=$(echo "$CREATED_AT" | sed 's/\.[0-9]*//' | sed 's/ [A-Z][A-Z]*$//')
CREATED_EPOCH=$(date -d "$CREATED_CLEAN" +%s 2>/dev/null || \
node -e 'const raw = process.argv[1] || ""; const base = raw.replace(/\.\d+/, "").replace(/\s+[A-Z]{2,4}$/, ""); const spaced = base.replace(/^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})\s+([+-]\d{2})(\d{2})$/, "$1T$2$3:$4"); const iso = base.replace(/^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})(Z|[+-]\d{2}:?\d{2})$/, "$1T$2$3"); const candidates = [raw, base, spaced, iso]; for (const candidate of candidates) { const ms = Date.parse(candidate); if (Number.isFinite(ms)) { console.log(Math.floor(ms / 1000)); process.exit(0); } } process.exit(1);' \
"$CREATED_AT" 2>/dev/null || \
python3 -c "import sys, datetime, re; ts = re.sub(r'\\.\d+', '', sys.argv[1]); ts = re.sub(r'\\s+[A-Z]{2,4}$', '', ts.strip()); print(int(datetime.datetime.strptime(ts, '%Y-%m-%d %H:%M:%S %z').timestamp()))" \
"$CREATED_AT" 2>/dev/null || echo 0)
NOW_EPOCH=$(date +%s)
LOCK_AGE=$((NOW_EPOCH - CREATED_EPOCH))
# 2026-05-05 Codex: dirty reboot / cancelled Actions can leave
# the Docker-network lock behind with no active build or push.
# Waiting the full 30m CD timeout keeps deploys queued even
# though no job is protected, so clear empty locks after 5m.
# 2026-06-18 Codex: 只靠 bracket pattern 仍會命中 lock-check
# bash/awk 自己的指令列；必須排除檢查器本身，取消後留下的
# empty lock network 才能在 5 分鐘後自清。
ACTIVE_DOCKER_WORK=$(ps -eo pid,args | awk '
$0 ~ /[d]ocker (build|push)|[b]uildx build/ &&
$0 !~ /ACTIVE_DOCKER_WORK/ &&
$0 !~ /awk/ &&
$0 !~ /ps -eo pid,args/ {print}
' || true)
if [ "$CREATED_EPOCH" -eq 0 ] && \
[ $((attempt * WAIT_SLEEP_SECONDS)) -gt $((EMPTY_LOCK_SECONDS * 2)) ] && \
[ -z "$ACTIVE_DOCKER_WORK" ]; then
echo "⚠️ Docker build lock has unparsable CreatedAt (${CREATED_AT}) and no active docker build/push after $((attempt * WAIT_SLEEP_SECONDS))s, removing ${LOCK_NAME}"
docker network rm "$LOCK_NAME" >/dev/null 2>&1 || true
continue
fi
if [ "$CREATED_EPOCH" -gt 0 ] && \
[ "$LOCK_AGE" -gt "$EMPTY_LOCK_SECONDS" ] && \
[ -z "$ACTIVE_DOCKER_WORK" ]; then
echo "⚠️ empty Docker build lock detected (age=${LOCK_AGE}s > ${EMPTY_LOCK_SECONDS}s, no active docker build/push), removing ${LOCK_NAME}"
docker network rm "$LOCK_NAME" >/dev/null 2>&1 || true
continue
fi
if [ "$CREATED_EPOCH" -gt 0 ] && \
[ "$LOCK_AGE" -gt "$STALE_SECONDS" ]; then
echo "⚠️ stale Docker build lock detected (age=${LOCK_AGE}s > ${STALE_SECONDS}s), removing ${LOCK_NAME}"
docker network rm "$LOCK_NAME" >/dev/null 2>&1 || true
continue
fi
fi
echo "⏳ Docker build lock busy (attempt ${attempt}/${WAIT_ATTEMPTS}); waiting ${WAIT_SLEEP_SECONDS}s..."
if [ "$attempt" -lt "$WAIT_ATTEMPTS" ]; then
sleep "$WAIT_SLEEP_SECONDS"
fi
done
echo "⚠️ timed out waiting for Docker build lock"
if [ "$LOCK_WARN_ONLY" = "1" ]; then
echo "⚠️ continuing without exclusive Docker build lock under commander controlled automation"
exit 0
fi
echo "❌ refusing to continue without Docker build lock"
exit 1
- name: Mirror Runtime Cache Images to Harbor
run: |
set -euo pipefail
prepare_deploy_key() {
mkdir -p "${HOME}/.ssh"
umask 077
local source_key="${AWOOOI_DEPLOY_SSH_KEY_PATH:-${HOME}/.ssh/deploy_key}"
if [ ! -r "${source_key}" ]; then
echo "BLOCKER runtime_image_mirror_deploy_key_missing"
exit 1
fi
if [ "${source_key}" != "${HOME}/.ssh/deploy_key" ]; then
cp "${source_key}" "${HOME}/.ssh/deploy_key"
fi
chmod 600 "${HOME}/.ssh/deploy_key"
}
prepare_deploy_key
CACHE_KNOWN_HOSTS="${HOME}/.ssh/runtime_cache_known_hosts"
ssh-keyscan -T 5 -t ed25519,rsa,ecdsa 192.168.0.120 \
> "${CACHE_KNOWN_HOSTS}" 2>/dev/null
test -s "${CACHE_KNOWN_HOSTS}" || {
echo "BLOCKER runtime_image_cache_host_key_unavailable"
exit 1
}
STAGING_TRACE_ID="aia-p0-006-02b-${GITHUB_RUN_ID:-${GITEA_RUN_NUMBER:-unknown}}"
STAGING_RECEIPT_PATH="/tmp/awoooi-runtime-image-staging-receipt.json"
rm -f "${STAGING_RECEIPT_PATH}"
python3 scripts/security/runtime_image_mirror_controller.py \
--policy config/security/runtime-image-mirror-staging-policy.json \
--trace-id "${STAGING_TRACE_ID}" \
stage \
--apply \
--receipt "${STAGING_RECEIPT_PATH}" \
--ssh-key "${HOME}/.ssh/deploy_key" \
--known-hosts "${CACHE_KNOWN_HOSTS}"
test -s "${STAGING_RECEIPT_PATH}"
echo "runtime_image_staging_stage=verified"
TRACE_ID="aia-p0-006-02a-${GITHUB_RUN_ID:-${GITEA_RUN_NUMBER:-unknown}}"
RECEIPT_PATH="/tmp/awoooi-runtime-image-mirror-receipt.json"
rm -f "${RECEIPT_PATH}"
python3 scripts/security/runtime_image_mirror_controller.py \
--policy config/security/runtime-image-mirror-policy.json \
--trace-id "${TRACE_ID}" \
mirror \
--apply \
--receipt "${RECEIPT_PATH}" \
--ssh-key "${HOME}/.ssh/deploy_key" \
--known-hosts "${CACHE_KNOWN_HOSTS}"
test -s "${RECEIPT_PATH}"
echo "runtime_image_mirror_stage=verified"
# ── API 鏡像建置（含 Layer Cache 加速）──────────────────────────────
# 2026-04-01 ogt: CACHE_BUST=git_sha 確保 src/ 和 models.json 層每次重建
# deps 層 (pip install) 仍可 cache → 加速；代碼/配置層強制失效
# 2026-05-05 Codex: host runner bootstrap installs docker-cli-buildx;
# keep BuildKit enabled because the web Dockerfile uses RUN --mount.
- name: Build and Push API
env:
DOCKER_BUILDKIT: "1"
run: |
run_docker_step() {
local label="$1"
local timeout_seconds="$2"
shift 2
echo "cd_docker_step_start=${label} timeout_seconds=${timeout_seconds}"
set +e
if command -v timeout >/dev/null 2>&1; then
timeout -k 30s "${timeout_seconds}s" "$@"
else
"$@"
fi
local rc=$?
set -e
echo "cd_docker_step_rc=${label}:${rc}"
if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
echo "BLOCKER cd_docker_step_timeout label=${label} timeout_seconds=${timeout_seconds}"
echo "NEXT_ACTION inspect_cd_runner_docker_build_push_or_registry_route_then_rerun_cd"
fi
return "$rc"
}
run_docker_push_step() {
local label="$1"
local timeout_seconds="$2"
shift 2
local max_attempts="${CD_DOCKER_PUSH_ATTEMPTS:-3}"
local sleep_seconds="${CD_DOCKER_PUSH_RETRY_SLEEP_SECONDS:-15}"
local attempt=1
local rc=1
while [ "${attempt}" -le "${max_attempts}" ]; do
echo "cd_docker_push_attempt=${label}:${attempt}/${max_attempts}"
if run_docker_step "${label}" "${timeout_seconds}" "$@"; then
rc=0
else
rc=$?
fi
if [ "${rc}" -eq 0 ]; then
return 0
fi
if [ "${attempt}" -ge "${max_attempts}" ]; then
echo "BLOCKER cd_docker_push_failed label=${label} attempts=${max_attempts}"
echo "NEXT_ACTION inspect_harbor_registry_push_500_or_storage_pressure_then_rerun_cd"
return "${rc}"
fi
echo "cd_docker_push_retry_sleep=${label}:${sleep_seconds}"
sleep "${sleep_seconds}"
attempt=$((attempt + 1))
done
return "${rc}"
}
run_docker_step api_build "${DOCKER_API_BUILD_TIMEOUT_SECONDS:-1200}" \
docker build -f apps/api/Dockerfile \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--cache-from ${{ env.HARBOR }}/awoooi/api:latest \
--build-arg CACHE_BUST=${{ github.sha }} \
-t ${{ env.HARBOR }}/awoooi/api:${{ github.sha }} \
-t ${{ env.HARBOR }}/awoooi/api:latest \
.
run_docker_push_step api_push_sha "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \
docker push ${{ env.HARBOR }}/awoooi/api:${{ github.sha }}
run_docker_push_step api_push_latest "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \
docker push ${{ env.HARBOR }}/awoooi/api:latest
API_IMAGE_DIGEST="$(
docker manifest inspect --verbose \
"${{ env.HARBOR }}/awoooi/api:${{ github.sha }}" | \
python3 -c '
import json
import re
import sys
payload = json.load(sys.stdin)
digest = payload.get("Descriptor", {}).get("digest", "")
if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
raise SystemExit("api_harbor_digest_invalid")
print(digest)
'
)"
docker manifest inspect \
"${{ env.HARBOR }}/awoooi/api@${API_IMAGE_DIGEST}" >/dev/null
echo "API_IMAGE_DIGEST=${API_IMAGE_DIGEST}" >> "${GITHUB_ENV}"
echo "api_harbor_digest_verified=${API_IMAGE_DIGEST}"
# 2026-03-31 ogt: 移除中間通知，減少訊息雜訊
# ── Web 鏡像建置（精準快取失效）──────────────────────────────
# 2026-03-30 ogt: NEXT_PUBLIC_* 必須用公網域名 (build-time 寫死)
# 2026-04-01 Claude Code: CACHE_BUST=git_sha 取代 --no-cache
# - deps 層 (pnpm install) 仍可 cache → 節省 ~2-3 min
# - COPY . . 以下由 CACHE_BUST 強制失效 → 業務邏輯/CSRF 等變更正確進入 bundle
# 2026-05-05 Codex: mirror API build mode; BuildKit required for cache mounts.
- name: Build and Push Web
env:
DOCKER_BUILDKIT: "1"
run: |
run_docker_step() {
local label="$1"
local timeout_seconds="$2"
shift 2
echo "cd_docker_step_start=${label} timeout_seconds=${timeout_seconds}"
set +e
if command -v timeout >/dev/null 2>&1; then
timeout -k 30s "${timeout_seconds}s" "$@"
else
"$@"
fi
local rc=$?
set -e
echo "cd_docker_step_rc=${label}:${rc}"
if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
echo "BLOCKER cd_docker_step_timeout label=${label} timeout_seconds=${timeout_seconds}"
echo "NEXT_ACTION inspect_cd_runner_docker_build_push_or_registry_route_then_rerun_cd"
fi
return "$rc"
}
run_docker_push_step() {
local label="$1"
local timeout_seconds="$2"
shift 2
local max_attempts="${CD_DOCKER_PUSH_ATTEMPTS:-3}"
local sleep_seconds="${CD_DOCKER_PUSH_RETRY_SLEEP_SECONDS:-15}"
local attempt=1
local rc=1
while [ "${attempt}" -le "${max_attempts}" ]; do
echo "cd_docker_push_attempt=${label}:${attempt}/${max_attempts}"
if run_docker_step "${label}" "${timeout_seconds}" "$@"; then
rc=0
else
rc=$?
fi
if [ "${rc}" -eq 0 ]; then
return 0
fi
if [ "${attempt}" -ge "${max_attempts}" ]; then
echo "BLOCKER cd_docker_push_failed label=${label} attempts=${max_attempts}"
echo "NEXT_ACTION inspect_harbor_registry_push_500_or_storage_pressure_then_rerun_cd"
return "${rc}"
fi
echo "cd_docker_push_retry_sleep=${label}:${sleep_seconds}"
sleep "${sleep_seconds}"
attempt=$((attempt + 1))
done
return "${rc}"
}
# 2026-07-03 Codex: run #4610 proved the Harbor DockerHub proxy
# cache path returned 401 for node:20-alpine. Keep the public tag
# until proxy-cache pull auth has a deploy-log receipt. Pull it with
# bounded retries and retag locally so transient DockerHub HEAD 500s
# do not block source-only UI/CD releases after one failed attempt.
prepare_web_node_base_image() {
local local_tag="awoooi-web-node-base:20-alpine"
local source_image="node:20-alpine"
local attempt
local pull_rc
for attempt in 1 2 3; do
echo "web_node_base_pull_candidate=${source_image} attempt=${attempt}"
set +e
if command -v timeout >/dev/null 2>&1; then
timeout -k 10s 120s docker pull "${source_image}"
else
docker pull "${source_image}"
fi
pull_rc=$?
set -e
echo "web_node_base_pull_rc=${source_image}:${pull_rc}"
if [ "${pull_rc}" -eq 0 ]; then
docker tag "${source_image}" "${local_tag}"
WEB_NODE_BASE_IMAGE="${local_tag}"
export WEB_NODE_BASE_IMAGE
echo "web_node_base_image_selected=${WEB_NODE_BASE_IMAGE}"
return 0
fi
sleep 5
done
echo "BLOCKER web_node_base_image_pull_failed"
echo "NEXT_ACTION inspect_dockerhub_route_or_harbor_proxy_auth_then_rerun_cd"
return 1
}
prepare_web_node_base_image
run_docker_step web_build "${DOCKER_WEB_BUILD_TIMEOUT_SECONDS:-1500}" \
docker build -f apps/web/Dockerfile \
--build-arg NODE_BASE_IMAGE="${WEB_NODE_BASE_IMAGE}" \
--build-arg NEXT_PUBLIC_API_URL=https://awoooi.wooo.work \
--build-arg CACHE_BUST=${{ github.sha }} \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--cache-from ${{ env.HARBOR }}/awoooi/web:latest \
-t ${{ env.HARBOR }}/awoooi/web:${{ github.sha }} \
-t ${{ env.HARBOR }}/awoooi/web:latest \
.
run_docker_push_step web_push_sha "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \
docker push ${{ env.HARBOR }}/awoooi/web:${{ github.sha }}
run_docker_push_step web_push_latest "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \
docker push ${{ env.HARBOR }}/awoooi/web:latest
WEB_IMAGE_DIGEST="$(
docker manifest inspect --verbose \
"${{ env.HARBOR }}/awoooi/web:${{ github.sha }}" | \
python3 -c '
import json
import re
import sys
payload = json.load(sys.stdin)
digest = payload.get("Descriptor", {}).get("digest", "")
if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
raise SystemExit("web_harbor_digest_invalid")
print(digest)
'
)"
docker manifest inspect \
"${{ env.HARBOR }}/awoooi/web@${WEB_IMAGE_DIGEST}" >/dev/null
echo "WEB_IMAGE_DIGEST=${WEB_IMAGE_DIGEST}" >> "${GITHUB_ENV}"
echo "web_harbor_digest_verified=${WEB_IMAGE_DIGEST}"
- name: Release Docker Build Lock
if: always()
run: |
if [ -n "${DOCKER_BUILD_LOCK:-}" ]; then
docker network rm "$DOCKER_BUILD_LOCK" >/dev/null 2>&1 || true
echo "✅ Docker build lock released: ${DOCKER_BUILD_LOCK}"
else
echo "⚡ no Docker build lock to release"
fi
# Production drift proved the original CREATE TABLE IF NOT EXISTS
# migration did not update an older CHECK constraint. Reconcile this
# one additive allowlist before rolling out the scanner and verify it
# independently without exposing connection details.
- name: Reconcile Asset Inventory Type Constraint
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
run: |
set -euo pipefail
if [ -z "${MIGRATION_DATABASE_URL:-}" ]; then
echo "::error::MIGRATION_DATABASE_URL secret not set in Gitea"
exit 1
fi
# The migration target is a routed LAN endpoint; the default bridge
# keeps the verifier compatible with the runner's userns-remap policy.
docker run --rm --network bridge -i \
-e DATABASE_URL \
-e MIGRATION_DATABASE_URL \
-v "$PWD/apps/api/migrations/asset_inventory_database_types_2026-07-14.sql:/tmp/asset_inventory_type.sql:ro" \
"${HARBOR}/awoooi/api:${{ github.sha }}" python - <<'PY'
import asyncio
import os
import re
from pathlib import Path
import asyncpg
MIGRATION = Path("/tmp/asset_inventory_type.sql").read_text(encoding="utf-8")
REQUIRED_TYPES = {
"host", "container", "k8s_workload", "k8s_resource", "database", "table",
"website", "api_endpoint", "package", "log_stream", "km_entry", "frontend",
"backend", "ci_pipeline", "gitea_repo", "monitoring_target", "secret", "volume",
"network", "certificate", "scheduled_job", "message_queue", "cache", "dashboard",
"ai_agent", "llm_model", "third_party_service", "backup_target",
}
def normalize_url(value: str) -> str:
return value.replace("postgresql+asyncpg://", "postgresql://", 1)
async def constraint_status(
connection: asyncpg.Connection,
) -> tuple[bool, str, bool]:
row = await connection.fetchrow(
"""
SELECT
constraint_row.convalidated,
pg_get_constraintdef(constraint_row.oid) AS definition,
EXISTS (
SELECT 1
FROM automation_operation_log
WHERE actor = 'gitea_cd_asset_schema_reconciler'
AND operation_type = 'asset_discovered'
AND status = 'success'
AND input ->> 'migration' =
'asset_inventory_database_types_2026-07-14'
) AS durable_receipt_present
FROM pg_constraint constraint_row
WHERE constraint_row.conrelid = 'asset_inventory'::regclass
AND constraint_row.conname = 'asset_inventory_type_valid'
"""
)
if row is None:
return False, "", False
return (
bool(row["convalidated"]),
str(row["definition"] or ""),
bool(row["durable_receipt_present"]),
)
def allowed_types(definition: str) -> set[str]:
return set(re.findall(r"'([^']+)'(?:::text)?", definition))
async def reconcile(url: str) -> tuple[bool, str, bool, bool]:
connection = await asyncpg.connect(normalize_url(url), timeout=10)
applied = False
try:
validated, definition, receipt_present = await constraint_status(connection)
if (
not validated
or allowed_types(definition) != REQUIRED_TYPES
or not receipt_present
):
async with connection.transaction():
await connection.execute(MIGRATION)
applied = True
validated, definition, receipt_present = await constraint_status(connection)
return validated, definition, receipt_present, applied
finally:
await connection.close()
async def main() -> None:
candidates = [
("migration_role", os.environ["MIGRATION_DATABASE_URL"]),
("owner_role", os.environ.get("DATABASE_URL", "")),
]
for index, (role, url) in enumerate(candidates):
if not url:
continue
try:
validated, definition, receipt_present, applied = await reconcile(url)
type_set = allowed_types(definition)
database_allowed = "database" in type_set
table_allowed = "table" in type_set
type_set_exact = type_set == REQUIRED_TYPES
print(f"asset_inventory_type_migration_role={role}")
print(f"asset_inventory_type_migration_applied={str(applied).lower()}")
print(f"database_type_allowed={str(database_allowed).lower()}")
print(f"table_type_allowed={str(table_allowed).lower()}")
print(f"constraint_type_set_exact={str(type_set_exact).lower()}")
print(f"constraint_validated={str(validated).lower()}")
print(f"durable_receipt_present={str(receipt_present).lower()}")
if not (
validated
and database_allowed
and table_allowed
and type_set_exact
and receipt_present
):
raise RuntimeError("asset_inventory_type_constraint_verifier_failed")
return
except asyncpg.PostgresError as exc:
permission_denied = getattr(exc, "sqlstate", "") == "42501"
has_owner_fallback = index == 0 and bool(candidates[1][1])
if permission_denied and has_owner_fallback:
print("asset_inventory_type_migration_owner_fallback=true")
continue
print(f"asset_inventory_type_migration_error_class={type(exc).__name__}")
raise
raise RuntimeError("asset_inventory_type_migration_no_usable_connection")
asyncio.run(main())
PY
# A valid unique index can still hide heap/index drift introduced by an
# older restore or repair. Quarantine only the non-canonical rows, rebuild
# the index transactionally, and prove the conflict-update path before
# rolling out the scanner.
- name: Repair Asset Inventory Canonical Integrity
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
run: |
set -euo pipefail
if [ -z "${MIGRATION_DATABASE_URL:-}" ]; then
echo "::error::MIGRATION_DATABASE_URL secret not set in Gitea"
exit 1
fi
docker run --rm --network bridge -i \
-e DATABASE_URL \
-e MIGRATION_DATABASE_URL \
-v "$PWD/apps/api/migrations/asset_inventory_canonical_integrity_2026-07-14.sql:/tmp/asset_inventory_integrity.sql:ro" \
"${HARBOR}/awoooi/api:${{ github.sha }}" python - <<'PY'
import asyncio
import json
import os
from pathlib import Path
import asyncpg
MIGRATION_NAME = "asset_inventory_canonical_integrity_2026-07-14"
MIGRATION = Path("/tmp/asset_inventory_integrity.sql").read_text(
encoding="utf-8"
)
def normalize_url(value: str) -> str:
return value.replace("postgresql+asyncpg://", "postgresql://", 1)
def receipt_valid(receipt: dict[str, object] | None) -> bool:
return bool(
receipt
and receipt.get("duplicate_group_count_after") == 0
and receipt.get("index_rebuilt") is True
and receipt.get("canonical_upsert_verified") is True
and receipt.get("data_rows_deleted") == 0
)
async def integrity_status(
connection: asyncpg.Connection,
) -> tuple[int, int, bool, dict[str, object] | None]:
row = await connection.fetchrow(
"""
WITH duplicate_groups AS (
SELECT COUNT(*) AS count
FROM (
SELECT asset_key
FROM asset_inventory
GROUP BY asset_key
HAVING COUNT(*) > 1
) AS grouped_duplicates
), latest_receipt AS (
SELECT output::text AS output
FROM automation_operation_log
WHERE actor = 'gitea_cd_asset_integrity_reconciler'
AND operation_type = 'asset_discovered'
AND status = 'success'
AND input ->> 'migration' = $1
ORDER BY created_at DESC
LIMIT 1
)
SELECT
duplicate_groups.count AS duplicate_group_count,
(
SELECT COUNT(*)
FROM asset_inventory
WHERE lifecycle_state = 'deprecated'
AND metadata ->> 'integrity_state' =
'quarantined_duplicate'
) AS quarantined_row_count,
COALESCE((
SELECT
index_row.indisvalid
AND index_row.indisready
AND index_row.indislive
AND index_row.indisunique
FROM pg_index AS index_row
WHERE index_row.indexrelid =
'asset_inventory_asset_key_key'::regclass
), FALSE) AS unique_index_healthy,
(SELECT output FROM latest_receipt) AS receipt_output
FROM duplicate_groups
""",
MIGRATION_NAME,
)
if row is None:
raise RuntimeError("asset_inventory_integrity_status_missing")
receipt_text = row["receipt_output"]
receipt = json.loads(receipt_text) if receipt_text else None
return (
int(row["duplicate_group_count"] or 0),
int(row["quarantined_row_count"] or 0),
bool(row["unique_index_healthy"]),
receipt,
)
async def reconcile(
url: str,
) -> tuple[int, int, bool, dict[str, object] | None, bool]:
connection = await asyncpg.connect(normalize_url(url), timeout=10)
applied = False
try:
# Do not let a corrupt-but-valid unique index hide duplicate
# heap rows from either the preflight or post-verifier.
await connection.execute("SET enable_indexscan TO off")
await connection.execute("SET enable_indexonlyscan TO off")
await connection.execute("SET enable_bitmapscan TO off")
duplicate_groups, _, index_healthy, receipt = (
await integrity_status(connection)
)
if (
duplicate_groups > 0
or not index_healthy
or not receipt_valid(receipt)
):
async with connection.transaction():
await connection.execute(MIGRATION)
applied = True
status = await integrity_status(connection)
return (*status, applied)
finally:
await connection.close()
async def main() -> None:
candidates = [
("migration_role", os.environ["MIGRATION_DATABASE_URL"]),
("owner_role", os.environ.get("DATABASE_URL", "")),
]
for index, (role, url) in enumerate(candidates):
if not url:
continue
try:
(
duplicate_groups,
quarantined_rows,
index_healthy,
receipt,
applied,
) = await reconcile(url)
verified = (
duplicate_groups == 0
and index_healthy
and receipt_valid(receipt)
)
print(f"asset_inventory_integrity_migration_role={role}")
print(
"asset_inventory_integrity_migration_applied="
f"{str(applied).lower()}"
)
print(f"duplicate_group_count={duplicate_groups}")
print(f"quarantined_row_count={quarantined_rows}")
print(f"unique_index_healthy={str(index_healthy).lower()}")
print(
"durable_integrity_receipt_present="
f"{str(receipt is not None).lower()}"
)
print(f"canonical_integrity_verified={str(verified).lower()}")
if not verified:
raise RuntimeError(
"asset_inventory_canonical_integrity_verifier_failed"
)
return
except asyncpg.PostgresError as exc:
permission_denied = getattr(exc, "sqlstate", "") == "42501"
has_owner_fallback = index == 0 and bool(candidates[1][1])
if permission_denied and has_owner_fallback:
print("asset_inventory_integrity_owner_fallback=true")
continue
print(
"asset_inventory_integrity_error_class="
f"{type(exc).__name__}"
)
raise
raise RuntimeError(
"asset_inventory_integrity_migration_no_usable_connection"
)
asyncio.run(main())
PY
# 2026-03-31 ogt: 移除中間通知
# 2026-03-31 ogt: P0-1 Secrets 自動注入 (ADR-035 強制)
# 2026-03-31 ogt: 加入 AI API Keys (修復 mock_fallback 問題)
- name: Inject K8s Secrets
env:
ARGOCD_API_TOKEN: ${{ secrets.ARGOCD_API_TOKEN }}
AWOOOI_GITEA_API_TOKEN: ${{ secrets.AWOOOI_GITEA_API_TOKEN }}
AWOOOI_GITEA_WEBHOOK_SECRET: ${{ secrets.AWOOOI_GITEA_WEBHOOK_SECRET }}
AWOOOP_OPERATOR_API_KEY: ${{ secrets.AWOOOP_OPERATOR_API_KEY }}
CLAUDE_API_KEY: ${{ secrets.CLAUDE_API_KEY }}
DATABASE_URL: ${{ secrets.DATABASE_URL }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
JWT_ALGORITHM: ${{ secrets.JWT_ALGORITHM }}
JWT_SECRET: ${{ secrets.JWT_SECRET }}
LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
NEMOTRON_BOT_TOKEN: ${{ secrets.NEMOTRON_BOT_TOKEN }}
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
OPENCLAW_BOT_TOKEN: ${{ secrets.OPENCLAW_BOT_TOKEN }}
OPENCLAW_TG_USER_WHITELIST: ${{ secrets.OPENCLAW_TG_USER_WHITELIST }}
REDIS_URL: ${{ secrets.REDIS_URL }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
SMTP_HOST: ${{ secrets.SMTP_HOST }}
SRE_GROUP_CHAT_ID: ${{ secrets.SRE_GROUP_CHAT_ID }}
TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
WEBHOOK_HMAC_SECRET: ${{ secrets.WEBHOOK_HMAC_SECRET }}
run: |
# 2026-06-30 Codex: do not inline action secret expressions in run scripts.
# Gitea renders the shell body into job logs before masking. Keep
# secret values in process env, then read by variable name only.
secret_b64_env() {
local env_name="$1"
if command -v python3.11 >/dev/null 2>&1; then
SECRET_ENV_NAME="${env_name}" python3.11 - <<'PY'
import base64
import os
data = os.environ.get(os.environ["SECRET_ENV_NAME"], "").encode()
data = data[:-1] if data.endswith(b"\n") else data
print(base64.b64encode(data).decode(), end="")
PY
elif command -v python3 >/dev/null 2>&1; then
SECRET_ENV_NAME="${env_name}" python3 - <<'PY'
import base64
import os
data = os.environ.get(os.environ["SECRET_ENV_NAME"], "").encode()
data = data[:-1] if data.endswith(b"\n") else data
print(base64.b64encode(data).decode(), end="")
PY
else
secret_value="$(printenv "${env_name}" || true)"
printf '%s' "${secret_value}" | base64 | tr -d '\n'
fi
}
prepare_deploy_key() {
mkdir -p "${HOME}/.ssh"
umask 077
local source_key="${AWOOOI_DEPLOY_SSH_KEY_PATH:-${HOME}/.ssh/deploy_key}"
if [ ! -r "${source_key}" ]; then
echo "❌ deploy ssh key file missing: ${source_key}" >&2
exit 1
fi
if [ "${source_key}" != "${HOME}/.ssh/deploy_key" ]; then
cp "${source_key}" "${HOME}/.ssh/deploy_key"
fi
chmod 600 "${HOME}/.ssh/deploy_key"
}
TG_BOT_TOKEN_B64="$(secret_b64_env TELEGRAM_BOT_TOKEN)"
TG_CHAT_ID_B64="$(secret_b64_env SRE_GROUP_CHAT_ID)"
NVIDIA_API_KEY_B64="$(secret_b64_env NVIDIA_API_KEY)"
GEMINI_API_KEY_B64="$(secret_b64_env GEMINI_API_KEY)"
LANGFUSE_PUBLIC_KEY_B64="$(secret_b64_env LANGFUSE_PUBLIC_KEY)"
LANGFUSE_SECRET_KEY_B64="$(secret_b64_env LANGFUSE_SECRET_KEY)"
TG_USER_WHITELIST_B64="$(secret_b64_env OPENCLAW_TG_USER_WHITELIST)"
SENTRY_AUTH_TOKEN_B64="$(secret_b64_env SENTRY_AUTH_TOKEN)"
GITEA_WEBHOOK_SECRET_B64="$(secret_b64_env AWOOOI_GITEA_WEBHOOK_SECRET)"
ARGOCD_API_TOKEN_B64="$(secret_b64_env ARGOCD_API_TOKEN)"
DATABASE_URL_B64="$(secret_b64_env DATABASE_URL)"
MIGRATION_DATABASE_URL_B64="$(secret_b64_env MIGRATION_DATABASE_URL)"
REDIS_URL_B64="$(secret_b64_env REDIS_URL)"
JWT_SECRET_B64="$(secret_b64_env JWT_SECRET)"
JWT_ALGORITHM_B64="$(secret_b64_env JWT_ALGORITHM)"
WEBHOOK_HMAC_SECRET_B64="$(secret_b64_env WEBHOOK_HMAC_SECRET)"
AWOOOP_OPERATOR_API_KEY_B64="$(secret_b64_env AWOOOP_OPERATOR_API_KEY)"
SENTRY_DSN_B64="$(secret_b64_env SENTRY_DSN)"
CLAUDE_API_KEY_B64="$(secret_b64_env CLAUDE_API_KEY)"
GITEA_API_TOKEN_B64="$(secret_b64_env AWOOOI_GITEA_API_TOKEN)"
NEMOTRON_BOT_TOKEN_B64="$(secret_b64_env NEMOTRON_BOT_TOKEN)"
OPENCLAW_BOT_TOKEN_B64="$(secret_b64_env OPENCLAW_BOT_TOKEN)"
SMTP_HOST_B64="$(secret_b64_env SMTP_HOST)"
SRE_GROUP_CHAT_ID_B64="$(secret_b64_env SRE_GROUP_CHAT_ID)"
# S1/S2: 統一命名 deploy_key，改用 ssh-keyscan 與強制 host key 驗證。
prepare_deploy_key
# 2026-05-13 Codex: keyscan must include ED25519 explicitly. Some
# OpenSSH builds otherwise record only RSA/ECDSA, then strict deploy
# SSH fails with "No ED25519 host key is known" after image push.
# 2026-06-13 Codex: keep deploy-time host keys in a dedicated file.
# The runner user's global known_hosts is shared by cold-start and
# backup checks for 120/188; overwriting it here caused strict SSH
# recovery gates to flap after every CD run.
DEPLOY_KNOWN_HOSTS="${HOME}/.ssh/deploy_known_hosts"
ssh-keyscan -T 5 -t ed25519,rsa,ecdsa "${K8S_SSH_HOST}" > "${DEPLOY_KNOWN_HOSTS}" 2>/dev/null
test -s "${DEPLOY_KNOWN_HOSTS}" || { echo "❌ K8S host keyscan failed: ${K8S_SSH_HOST}"; exit 1; }
SSH_OPTS="-i ${HOME}/.ssh/deploy_key -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${DEPLOY_KNOWN_HOSTS} -o ConnectTimeout=10"
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" << SECRETS
set -e
K8S_API_SERVER="${{ env.K8S_API_SERVER }}"
KUBECTL="sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=\${K8S_API_SERVER}"
# 注入 Telegram Secrets (ADR-035 鐵律)
# 2026-06-12 Codex: OPENCLAW_TG_CHAT_ID 僅作舊欄位相容，
# 實際值必須與 SRE_GROUP_CHAT_ID 一致，避免正式告警旁路到其他群組。
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/OPENCLAW_TG_BOT_TOKEN","value":"${TG_BOT_TOKEN_B64}"},
{"op":"add","path":"/data/OPENCLAW_TG_CHAT_ID","value":"${TG_CHAT_ID_B64}"}
]' || { echo "❌ Telegram Secrets patch 失敗 — ADR-035 鐵律"; exit 1; }
# 2026-03-31 ogt: 注入 AI API Keys (修復 NVIDIA/Gemini mock_fallback)
# 2026-04-01 Claude Code: base64 -w 0 防止長 key 換行破壞 JSON
# NVIDIA NIM (免費 tier)
if [ -n "${NVIDIA_API_KEY_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/NVIDIA_API_KEY","value":"${NVIDIA_API_KEY_B64}"}
]' && echo "✅ NVIDIA_API_KEY 已注入" || echo "⚠️ NVIDIA_API_KEY patch 失敗"
else
echo "⚠️ NVIDIA_API_KEY 未設定，跳過"
fi
# Gemini (備援)
if [ -n "${GEMINI_API_KEY_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/GEMINI_API_KEY","value":"${GEMINI_API_KEY_B64}"}
]' && echo "✅ GEMINI_API_KEY 已注入" || echo "⚠️ GEMINI_API_KEY patch 失敗"
else
echo "⚠️ GEMINI_API_KEY 未設定，跳過"
fi
# 2026-04-01 Claude Code: Langfuse LLMOps keys (補齊 CD 注入，之前只有手動設定)
if [ -n "${LANGFUSE_PUBLIC_KEY_B64}" ] && [ -n "${LANGFUSE_SECRET_KEY_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/LANGFUSE_PUBLIC_KEY","value":"${LANGFUSE_PUBLIC_KEY_B64}"},
{"op":"add","path":"/data/LANGFUSE_SECRET_KEY","value":"${LANGFUSE_SECRET_KEY_B64}"}
]' && echo "✅ LANGFUSE keys 已注入" || echo "⚠️ LANGFUSE keys patch 失敗"
else
echo "⚠️ LANGFUSE_PUBLIC_KEY/SECRET_KEY 未設定，跳過 (現有 K8s secret 值維持不變)"
fi
# 2026-04-02 Claude Code: Telegram Whitelist (授權簽核用戶 ID)
if [ -n "${TG_USER_WHITELIST_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/OPENCLAW_TG_USER_WHITELIST","value":"${TG_USER_WHITELIST_B64}"}
]' && echo "✅ TG_USER_WHITELIST 已注入" || echo "⚠️ TG_USER_WHITELIST patch 失敗"
fi
# Phase O-4.1 2026-04-02: Sentry Auth Token (Wave A.1 ADR-037)
if [ -n "${SENTRY_AUTH_TOKEN_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/SENTRY_AUTH_TOKEN","value":"${SENTRY_AUTH_TOKEN_B64}"}
]' && echo "✅ SENTRY_AUTH_TOKEN 已注入" || echo "⚠️ SENTRY_AUTH_TOKEN patch 失敗"
else
echo "⚠️ SENTRY_AUTH_TOKEN 未設定，Sentry Comment API 將跳過"
fi
# ADR-059 2026-04-05 Claude Code: Gitea Webhook Secret
if [ -n "${GITEA_WEBHOOK_SECRET_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/GITEA_WEBHOOK_SECRET","value":"${GITEA_WEBHOOK_SECRET_B64}"}
]' && echo "✅ GITEA_WEBHOOK_SECRET 已注入" || echo "⚠️ GITEA_WEBHOOK_SECRET patch 失敗"
else
echo "⚠️ GITEA_WEBHOOK_SECRET 未設定，Gitea Webhook 簽章驗證將在 prod 失效"
fi
# MCP Phase 3: ArgoCD API Token (2026-04-11 Claude Sonnet 4.6)
if [ -n "${ARGOCD_API_TOKEN_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/ARGOCD_API_TOKEN","value":"${ARGOCD_API_TOKEN_B64}"}
]' && echo "✅ ARGOCD_API_TOKEN 已注入" || echo "⚠️ ARGOCD_API_TOKEN patch 失敗"
else
echo "⚠️ ARGOCD_API_TOKEN 未設定，ArgoCD MCP 將使用空 token"
fi
# ============================================================================
# ADR-090-B 2026-04-18 ogt + Claude Opus 4.7: L3-only 升級 L2（13 個 key）
# ============================================================================
# 目的: 消滅「只存 K8s etcd 單點」的災難盲區，Gitea Secret 成為正式真相來源
# 注意: 每個 block 與上方維持相同結構（if guard + base64 -w 0 + json patch）
# DATABASE_URL — PG 應用連線串（2026-04-18 輪替）
if [ -n "${DATABASE_URL_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/DATABASE_URL","value":"${DATABASE_URL_B64}"}
]' && echo "✅ DATABASE_URL 已注入" || echo "⚠️ DATABASE_URL patch 失敗"
else
echo "⚠️ DATABASE_URL 未設定，awoooi-api 將無法連 PG"
fi
# MIGRATION_DATABASE_URL — CI migration 用 awoooi_migrator 限權帳號（ADR-090-B）
if [ -n "${MIGRATION_DATABASE_URL_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/MIGRATION_DATABASE_URL","value":"${MIGRATION_DATABASE_URL_B64}"}
]' && echo "✅ MIGRATION_DATABASE_URL 已注入" || echo "⚠️ MIGRATION_DATABASE_URL patch 失敗"
fi
# REDIS_URL — Redis 連線（6380 on 188）
if [ -n "${REDIS_URL_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/REDIS_URL","value":"${REDIS_URL_B64}"}
]' && echo "✅ REDIS_URL 已注入" || echo "⚠️ REDIS_URL patch 失敗"
else
echo "⚠️ REDIS_URL 未設定"
fi
# JWT_SECRET / JWT_ALGORITHM — API 認證
if [ -n "${JWT_SECRET_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/JWT_SECRET","value":"${JWT_SECRET_B64}"}
]' && echo "✅ JWT_SECRET 已注入" || echo "⚠️ JWT_SECRET patch 失敗"
fi
if [ -n "${JWT_ALGORITHM_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/JWT_ALGORITHM","value":"${JWT_ALGORITHM_B64}"}
]' && echo "✅ JWT_ALGORITHM 已注入" || echo "⚠️ JWT_ALGORITHM patch 失敗"
fi
# WEBHOOK_HMAC_SECRET — Alertmanager webhook HMAC 簽章
if [ -n "${WEBHOOK_HMAC_SECRET_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/WEBHOOK_HMAC_SECRET","value":"${WEBHOOK_HMAC_SECRET_B64}"}
]' && echo "✅ WEBHOOK_HMAC_SECRET 已注入" || echo "⚠️ WEBHOOK_HMAC_SECRET patch 失敗"
fi
# AWOOOP_OPERATOR_API_KEY — AwoooP Operator mutation endpoints
if [ -n "${AWOOOP_OPERATOR_API_KEY_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/AWOOOP_OPERATOR_API_KEY","value":"${AWOOOP_OPERATOR_API_KEY_B64}"}
]' && echo "✅ AWOOOP_OPERATOR_API_KEY 已注入" || echo "⚠️ AWOOOP_OPERATOR_API_KEY patch 失敗"
fi
# SENTRY_DSN — Sentry 錯誤追蹤（不是 auth token）
if [ -n "${SENTRY_DSN_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/SENTRY_DSN","value":"${SENTRY_DSN_B64}"}
]' && echo "✅ SENTRY_DSN 已注入" || echo "⚠️ SENTRY_DSN patch 失敗"
fi
# CLAUDE_API_KEY — Claude 備援 LLM
if [ -n "${CLAUDE_API_KEY_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/CLAUDE_API_KEY","value":"${CLAUDE_API_KEY_B64}"}
]' && echo "✅ CLAUDE_API_KEY 已注入" || echo "⚠️ CLAUDE_API_KEY patch 失敗"
fi
# GITEA_API_TOKEN — Gitea API Token（從 AWOOOI_GITEA_API_TOKEN 映射）
if [ -n "${GITEA_API_TOKEN_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/GITEA_API_TOKEN","value":"${GITEA_API_TOKEN_B64}"}
]' && echo "✅ GITEA_API_TOKEN 已注入" || echo "⚠️ GITEA_API_TOKEN patch 失敗"
fi
# NEMOTRON_BOT_TOKEN / OPENCLAW_BOT_TOKEN — 多 Bot 架構
if [ -n "${NEMOTRON_BOT_TOKEN_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/NEMOTRON_BOT_TOKEN","value":"${NEMOTRON_BOT_TOKEN_B64}"}
]' && echo "✅ NEMOTRON_BOT_TOKEN 已注入" || echo "⚠️ NEMOTRON_BOT_TOKEN patch 失敗"
fi
if [ -n "${OPENCLAW_BOT_TOKEN_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/OPENCLAW_BOT_TOKEN","value":"${OPENCLAW_BOT_TOKEN_B64}"}
]' && echo "✅ OPENCLAW_BOT_TOKEN 已注入" || echo "⚠️ OPENCLAW_BOT_TOKEN patch 失敗"
fi
# SMTP_HOST / SRE_GROUP_CHAT_ID
if [ -n "${SMTP_HOST_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/SMTP_HOST","value":"${SMTP_HOST_B64}"}
]' && echo "✅ SMTP_HOST 已注入" || echo "⚠️ SMTP_HOST patch 失敗"
fi
if [ -n "${SRE_GROUP_CHAT_ID_B64}" ]; then
\$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
{"op":"add","path":"/data/SRE_GROUP_CHAT_ID","value":"${SRE_GROUP_CHAT_ID_B64}"}
]' && echo "✅ SRE_GROUP_CHAT_ID 已注入" || echo "⚠️ SRE_GROUP_CHAT_ID patch 失敗"
fi
# 2026-04-06 Claude Code: Sprint 3 T2 — known_hosts Secret (Security Fix A1)
# 替換關閉 host key 驗證的舊做法，讓 SSH 修復路徑使用已知主機指紋。
# asyncssh reads /etc/ssh-mcp/known_hosts and requires a non-empty
# OpenSSH known_hosts file. Keep hosts unhashed so both asyncssh and
# CLI diagnostics can trust the same secret.
# 2026-05-02 ogt + Claude Sonnet 4.6: 加主機完整性檢查
# 根因：partial scan（如 110 timeout、其他成功）會讓 [-s file] 通過、
# 後續 patch 推進缺漏的 known_hosts → asyncssh 拒所有 SSH。
# 修法：scan 完驗證所有 allowlisted 主機都在；缺任何一台就 abort，
# 不能覆蓋現有 secret，防止 production SSH 自動修復路徑癱瘓。
: > /tmp/known_hosts_repair
: > /tmp/known_hosts_scan_err
EXPECTED_HOSTS=5
PRESENT=0
for scan_attempt in 1 2 3; do
ssh-keyscan -T 5 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \
>> /tmp/known_hosts_repair 2>>/tmp/known_hosts_scan_err || true
sort -u /tmp/known_hosts_repair -o /tmp/known_hosts_repair
PRESENT=0
for ip in 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then
PRESENT=\$((PRESENT + 1))
fi
done
if [ "\$PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
break
fi
sleep "\$scan_attempt"
done
for ip in 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then
:
else
echo "⚠️ ssh-keyscan 缺主機 \${ip}"
fi
done
if [ "\$PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
\$KUBECTL create secret generic awoooi-repair-known-hosts \
-n awoooi-prod \
--from-file=known_hosts=/tmp/known_hosts_repair \
--dry-run=client -o yaml | \$KUBECTL apply -f - \
&& echo "✅ awoooi-repair-known-hosts Secret 已建立/更新" \
|| echo "⚠️ awoooi-repair-known-hosts Secret 建立失敗 (非致命)"
KNOWN_HOSTS_B64=\$(base64 -w 0 /tmp/known_hosts_repair)
\$KUBECTL patch secret ssh-mcp-key -n awoooi-prod --type=merge \
-p="{\"data\":{\"known_hosts\":\"\${KNOWN_HOSTS_B64}\"}}" \
&& echo "✅ ssh-mcp-key known_hosts 已更新（5 台主機完整）" \
|| { echo "❌ ssh-mcp-key known_hosts 更新失敗，停止部署"; exit 1; }
rm -f /tmp/known_hosts_repair /tmp/known_hosts_scan_err
else
# A transient guest SSH outage must not overwrite trust data, but it
# also must not blindly green-light the deploy. Decode only on the
# K3s controller, verify all five public host entries, then delete
# the temporary file without printing its contents.
EXISTING_KNOWN_HOSTS=/tmp/known_hosts_existing
EXISTING_PRESENT=0
if \$KUBECTL get secret ssh-mcp-key -n awoooi-prod \
-o jsonpath='{.data.known_hosts}' | base64 -d > "\$EXISTING_KNOWN_HOSTS"; then
for ip in 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
if grep -qE "^\${ip}[[:space:]]" "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then
EXISTING_PRESENT=\$((EXISTING_PRESENT + 1))
fi
done
fi
rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err
if [ "\$EXISTING_PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
echo "⚠️ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key verified 5/5 and preserved"
else
echo "❌ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key coverage \${EXISTING_PRESENT}/\${EXPECTED_HOSTS}"
exit 1
fi
fi
echo "✅ 所有 Secrets 注入完成"
SECRETS
# AIA-P0-011: record only Secret object pre-existence, never Secret data.
# The bootstrap streams CSPRNG credentials directly to PostgreSQL and
# kubectl stdin, then proves each identity with a no-write preflight.
WORKLOAD_DB_SECRET_PREEXISTENCE_FILE="/tmp/awoooi-workload-db-secret-preexistence"
: > "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE"
for secret_name in awoooi-api-db awoooi-worker-db awoooi-broker-db; do
secret_existed=$(ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; if \$KUBECTL get secret '${secret_name}' -n awoooi-prod >/dev/null 2>&1; then printf 1; else printf 0; fi")
printf '%s=%s\n' "$secret_name" "$secret_existed" \
>> "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE"
done
cat scripts/ops/awoooi-workload-db-identity-bootstrap.sh | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"AWOOOI_KUBECTL_SUDO=true AWOOOI_KUBECONFIG=/etc/rancher/k3s/k3s.yaml AWOOOI_KUBERNETES_SERVER='${{ env.K8S_API_SERVER }}' AWOOOI_RUN_ID='${GITEA_RUN_NUMBER:-cd}' bash -s -- apply"
echo "✅ workload DB identities, Secret projections, and preflights ready"
# 2026-04-11 Claude Sonnet 4.6 (Sprint B-3 ADR-069):
# Deploy 改為 ArgoCD GitOps 模式：更新 kustomization.yaml → git push [skip ci] → ArgoCD sync
# 舊做法 (kubectl set image) 與 ArgoCD selfHeal 衝突 — ArgoCD 會 revert 任何直接 kubectl 操作
# 新做法流程：
# 1. 更新 kustomization.yaml image tag（用 kustomize edit set image）
# 2. Apply ConfigMap/ServiceRegistry（不含 Deployment，由 ArgoCD 管）
# 3. git commit [skip ci] + push → 觸發 ArgoCD automated sync
# 4. 等待 ArgoCD sync + rollout 完成
# 5. Health Check
- name: Deploy to K8s (ArgoCD GitOps)
env:
CD_PUSH_TOKEN: ${{ secrets.CD_PUSH_TOKEN }}
run: |
prepare_deploy_key() {
mkdir -p "${HOME}/.ssh"
umask 077
local source_key="${AWOOOI_DEPLOY_SSH_KEY_PATH:-${HOME}/.ssh/deploy_key}"
if [ ! -r "${source_key}" ]; then
echo "❌ deploy ssh key file missing: ${source_key}" >&2
exit 1
fi
if [ "${source_key}" != "${HOME}/.ssh/deploy_key" ]; then
cp "${source_key}" "${HOME}/.ssh/deploy_key"
fi
chmod 600 "${HOME}/.ssh/deploy_key"
}
mkdir -p ~/.ssh
prepare_deploy_key
# 2026-05-13 Codex: mirror Inject K8s Secrets host-key handling so the
# deploy job never reaches SSH with a known_hosts file missing ED25519.
# 2026-06-13 Codex: use the deploy-only known_hosts file so this
# stage cannot wipe cold-start/backup host trust for 120/188.
DEPLOY_KNOWN_HOSTS="${HOME}/.ssh/deploy_known_hosts"
ssh-keyscan -T 5 -t ed25519,rsa,ecdsa "${K8S_SSH_HOST}" > "${DEPLOY_KNOWN_HOSTS}" 2>/dev/null
test -s "${DEPLOY_KNOWN_HOSTS}" || { echo "❌ K8S host keyscan failed: ${K8S_SSH_HOST}"; exit 1; }
SSH_OPTS="-i ${HOME}/.ssh/deploy_key -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${DEPLOY_KNOWN_HOSTS} -o ConnectTimeout=10"
export IMAGE_TAG="${{ github.sha }}"
HARBOR=192.168.0.110:5000
if ! printf '%s\n' "${API_IMAGE_DIGEST:-}" | \
grep -Eq '^sha256:[0-9a-f]{64}$'; then
echo "BLOCKER api_harbor_digest_missing"
exit 1
fi
if ! printf '%s\n' "${WEB_IMAGE_DIGEST:-}" | \
grep -Eq '^sha256:[0-9a-f]{64}$'; then
echo "BLOCKER web_harbor_digest_missing"
exit 1
fi
# ─── Step 1: Apply NetworkPolicy + ConfigMap + ServiceRegistry ───
# Preserve only live Deployment specs and Secret object existence.
# Secret data is never read or copied into the rollback artifact.
WORKLOAD_DB_ROLLBACK_FILE="/tmp/awoooi-workload-db-deployment-rollback.yaml"
WORKLOAD_DB_SECRET_PREEXISTENCE_FILE="/tmp/awoooi-workload-db-secret-preexistence"
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL get deployment awoooi-api awoooi-worker awoooi-ansible-executor-broker -n awoooi-prod -o json" \
| python3 -c '
import json
import sys
source = json.load(sys.stdin)
items = [
{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": item["metadata"]["name"],
"namespace": item["metadata"]["namespace"],
"labels": item["metadata"].get("labels", {}),
},
"spec": item["spec"],
}
for item in source["items"]
]
json.dump({"apiVersion": "v1", "kind": "List", "items": items}, sys.stdout)
' > "$WORKLOAD_DB_ROLLBACK_FILE"
test -s "$WORKLOAD_DB_ROLLBACK_FILE"
if [ ! -s "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" ]; then
: > "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE"
for secret_name in awoooi-api-db awoooi-worker-db awoooi-broker-db; do
secret_existed=$(ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; if \$KUBECTL get secret '${secret_name}' -n awoooi-prod >/dev/null 2>&1; then printf 1; else printf 0; fi")
printf '%s=%s\n' "$secret_name" "$secret_existed" \
>> "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE"
done
fi
restore_workload_database_projection() {
echo "⏪ restoring previous AWOOOI workload database projection"
cat "$WORKLOAD_DB_ROLLBACK_FILE" | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"
for deployment_name in awoooi-api awoooi-worker awoooi-ansible-executor-broker; do
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL rollout status deployment/${deployment_name} -n awoooi-prod --timeout=180s"
done
disable_workloads=()
while IFS='=' read -r secret_name secret_existed; do
[ "$secret_existed" = "0" ] || continue
case "$secret_name" in
awoooi-api-db) disable_workloads+=(api) ;;
awoooi-worker-db) disable_workloads+=(worker) ;;
awoooi-broker-db) disable_workloads+=(broker) ;;
esac
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL delete secret '${secret_name}' -n awoooi-prod --ignore-not-found=true"
done < "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE"
if [ "${#disable_workloads[@]}" -gt 0 ]; then
cat scripts/ops/awoooi-workload-db-identity-bootstrap.sh | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"AWOOOI_KUBECTL_SUDO=true AWOOOI_KUBECONFIG=/etc/rancher/k3s/k3s.yaml AWOOOI_KUBERNETES_SERVER='${{ env.K8S_API_SERVER }}' bash -s -- disable ${disable_workloads[*]}"
fi
}
# NetworkPolicy is intentionally outside kustomize because commonLabels
# mutates nested selectors. The checkout is depth=1, so snapshot live
# policy truth and remove server-only metadata before any mutation.
NETWORK_POLICY_ROLLBACK_FILE="/tmp/awoooi-network-policy-rollback.json"
NETWORK_POLICY_CHANGE_ACTIVE="0"
BROKER_POLICY_NAME="allow-ansible-executor-broker-ssh-egress"
BROKER_POLICY_EXISTED=$(ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; if \$KUBECTL get networkpolicy '$BROKER_POLICY_NAME' -n awoooi-prod >/dev/null 2>&1; then printf 1; else printf 0; fi")
NETWORK_POLICY_NAMES=(
default-deny-all
allow-nginx-ingress
allow-required-egress
deny-legacy-access
)
if [ "$BROKER_POLICY_EXISTED" = "1" ]; then
NETWORK_POLICY_NAMES+=("$BROKER_POLICY_NAME")
fi
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL get networkpolicy ${NETWORK_POLICY_NAMES[*]} -n awoooi-prod -o json" \
| python3 -c '
import json
import sys
source = json.load(sys.stdin)
items = [
{
"apiVersion": item["apiVersion"],
"kind": item["kind"],
"metadata": {
"name": item["metadata"]["name"],
"namespace": item["metadata"]["namespace"],
},
"spec": item["spec"],
}
for item in source["items"]
]
json.dump({"apiVersion": "v1", "kind": "List", "items": items}, sys.stdout)
' \
> "$NETWORK_POLICY_ROLLBACK_FILE"
test -s "$NETWORK_POLICY_ROLLBACK_FILE"
restore_network_policy() {
echo "⏪ restoring previous AWOOOI NetworkPolicy contract"
if cat "$NETWORK_POLICY_ROLLBACK_FILE" | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"; then
restore_status="0"
else
restore_status="$?"
fi
if [ "$BROKER_POLICY_EXISTED" = "0" ]; then
if ! ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL delete networkpolicy '$BROKER_POLICY_NAME' -n awoooi-prod --ignore-not-found=true"; then
restore_status="1"
fi
fi
if [ "$restore_status" = "0" ]; then
NETWORK_POLICY_CHANGE_ACTIVE="0"
fi
return "$restore_status"
}
restore_deploy_controls() {
set +e
restore_workload_database_projection
workload_restore_exit=$?
restore_network_policy
network_restore_exit=$?
set -e
if [ "$workload_restore_exit" -ne 0 ] \
|| [ "$network_restore_exit" -ne 0 ]; then
echo "❌ deploy rollback incomplete: workload=${workload_restore_exit} network=${network_restore_exit}" >&2
return 1
fi
}
rollback_deploy_controls_on_exit() {
exit_code=$?
trap - EXIT
if [ "$exit_code" -ne 0 ] \
&& [ "$NETWORK_POLICY_CHANGE_ACTIVE" = "1" ]; then
echo "⏪ deploy exited before the terminal boundary verifier"
set +e
restore_deploy_controls
restore_exit=$?
set -e
if [ "$restore_exit" -ne 0 ]; then
echo "❌ deploy control rollback failed"
fi
fi
exit "$exit_code"
}
trap rollback_deploy_controls_on_exit EXIT
cat k8s/awoooi-prod/02-network-policy.yaml | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply --dry-run=server -f - >/dev/null"
if ! cat k8s/awoooi-prod/02-network-policy.yaml | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"; then
restore_deploy_controls || true
exit 1
fi
NETWORK_POLICY_CHANGE_ACTIVE="1"
echo "✅ NetworkPolicy 已受控更新"
cat k8s/awoooi-prod/04-configmap.yaml | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"
echo "✅ ConfigMap 已更新"
cat k8s/awoooi-prod/15-service-registry-configmap.yaml | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"
echo "✅ Service Registry ConfigMap 已更新"
# ─── Step 2: 更新 kustomization.yaml immutable image digest ───
# kustomize is an immutable dependency of the internal runner image.
# GitHub is frozen, so CD must never download tools from public GitHub.
export PATH="${HOME}/.local/bin:${PATH}"
command -v kustomize >/dev/null 2>&1 || {
echo "❌ internal runner image is missing required kustomize"
exit 1
}
cd k8s/awoooi-prod
# Source SHA remains in AWOOOI_BUILD_COMMIT_SHA; workload identity is
# pinned to the independently read-back Harbor manifest digest.
kustomize edit set image \
192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER=${HARBOR}/awoooi/api@${API_IMAGE_DIGEST}
kustomize edit set image \
192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER=${HARBOR}/awoooi/web@${WEB_IMAGE_DIGEST}
cd ../..
# 2026-06-29 Codex: API deploy readback uses AWOOOI_BUILD_COMMIT_SHA
# to compare runtime image/source truth. Keep it in the same deploy
# marker commit as kustomization.yaml so the production Workbench does
# not depend on stale committed snapshots.
python3 - <<'PY'
import os
import re
from pathlib import Path
image_tag = os.environ["IMAGE_TAG"]
def replace_env_value(source: str, env_name: str) -> str:
pattern = (
rf'(\n\s+- name: {re.escape(env_name)}\n'
r'(?:\s+# [^\n]*\n)*'
r'\s+value: ")[^"]*(")'
)
if not re.search(pattern, source):
raise SystemExit(f"{env_name} env block not found")
return re.sub(
pattern,
lambda match: f"{match.group(1)}{image_tag}{match.group(2)}",
source,
count=1,
)
manifests = {
Path("k8s/awoooi-prod/06-deployment-api.yaml"): (
"AWOOOI_BUILD_COMMIT_SHA",
"AWOOOI_DESIRED_API_IMAGE_TAG",
),
Path("k8s/awoooi-prod/08-deployment-worker.yaml"): (
"AWOOOI_BUILD_COMMIT_SHA",
),
Path(
"k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml"
): ("AWOOOI_BUILD_COMMIT_SHA",),
}
for path, env_names in manifests.items():
text = path.read_text(encoding="utf-8")
for env_name in env_names:
text = replace_env_value(text, env_name)
path.write_text(text, encoding="utf-8")
PY
# ─── Step 3: git commit [skip ci] + push → 觸發 ArgoCD sync ───
git config user.email "cd@awoooi.internal"
git config user.name "AWOOOI CD"
git add \
k8s/awoooi-prod/kustomization.yaml \
k8s/awoooi-prod/06-deployment-api.yaml \
k8s/awoooi-prod/08-deployment-worker.yaml \
k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml
DEPLOY_REVISION=""
git diff --cached --quiet && echo "⚡ kustomization.yaml 無變化，跳過 push" || {
git commit -m "chore(cd): deploy ${IMAGE_TAG::7} [skip ci]"
# 用 token 推送（避免 SSH key 需要額外設定 push 權限）
git remote remove gitea 2>/dev/null || true
git remote add gitea "http://wooo:${CD_PUSH_TOKEN}@192.168.0.110:3001/wooo/awoooi.git"
# 先 rebase 避免 non-fast-forward (其他 commit 在 CI 期間已推入)
# 2026-04-17 ogt: -X theirs — kustomization.yaml 衝突時採用當次部署的 image tag
git fetch gitea main
git rebase -X theirs gitea/main
DEPLOY_REVISION=$(git rev-parse HEAD)
git push gitea main
echo "✅ kustomization.yaml 已 push，等待 ArgoCD sync 到 ${DEPLOY_REVISION:0:8}..."
}
# ─── Step 4: 等待 ArgoCD sync + rollout ───
ROLLOUT_LOG="$(mktemp)"
set +e
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"EXPECTED_REVISION='${DEPLOY_REVISION}' SOURCE_REVISION='${IMAGE_TAG}' WORKFLOW_RUN_ID='${GITHUB_RUN_ID:-unknown}' bash -s" 2>&1 << 'ARGOCD_WAIT' | tee "$ROLLOUT_LOG"
set -e
K8S_API_SERVER="${{ env.K8S_API_SERVER }}"
KUBECTL="sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${K8S_API_SERVER}"
RISK_FILE="$(mktemp)"
UNKNOWN_STATUS_COUNT=0
HEALTH_FAILURE_COUNT=0
DEPLOY_STAGE="argocd_sync"
DEPLOY_TERMINAL_RECORDED=0
# The API probes permit 305 seconds for cold-start DB bootstrap plus
# 20 seconds for readiness. Keep rollout above that bounded budget.
ROLLOUT_TIMEOUT_SECONDS=360
record_rollout_risk() {
local message="$1"
printf '%s\n' "$message" >> "$RISK_FILE"
echo "⚠️ Rollout risk observed: $message" >&2
}
emit_deploy_terminal_receipt() {
local status="$1"
local stage="$2"
local workload="$3"
if [ "$DEPLOY_TERMINAL_RECORDED" = "1" ]; then
return 0
fi
printf '%s\n' \
"AWOOOI_DEPLOY_TERMINAL_RECEIPT=status=${status};stage=${stage};workload=${workload};source_revision=${SOURCE_REVISION};workflow_run_id=${WORKFLOW_RUN_ID};verifier_no_write=true;rollback_required=true"
DEPLOY_TERMINAL_RECORDED=1
}
emit_rollout_evidence() {
local exit_code=$?
if [ "$exit_code" -ne 0 ]; then
emit_deploy_terminal_receipt \
"blocked_with_safe_next_action" "$DEPLOY_STAGE" "unknown"
fi
if [ -s "$RISK_FILE" ]; then
local summary
local kubectl_count
kubectl_count=$(grep -c '^argocd_.*_query_failed=' "$RISK_FILE" 2>/dev/null || true)
summary=$(tr '\n' '; ' < "$RISK_FILE" | sed 's/[[:cntrl:]]//g' | cut -c1-700)
echo "AWOOOI_ROLLOUT_RISK=1"
echo "AWOOOI_ROLLOUT_SUMMARY=unknown_status_count=${UNKNOWN_STATUS_COUNT}; health_failure_count=${HEALTH_FAILURE_COUNT}; kubectl_failure_count=${kubectl_count}; ${summary}"
else
echo "AWOOOI_ROLLOUT_RISK=0"
fi
rm -f "$RISK_FILE"
return "$exit_code"
}
trap emit_rollout_evidence EXIT
app_field() {
local jsonpath="$1"
local label="$2"
local output
local status
local kubectl_seen
set +e
output=$($KUBECTL get application awoooi-prod -n argocd -o jsonpath="$jsonpath" 2>&1)
status=$?
set -e
if [ "$status" -ne 0 ]; then
kubectl_seen=$(grep -c '^argocd_.*_query_failed=' "$RISK_FILE" 2>/dev/null || true)
if [ "$kubectl_seen" -lt 3 ]; then
record_rollout_risk "argocd_${label}_query_failed=$(echo "$output" | head -c 180)"
fi
printf 'Unknown'
return 0
fi
printf '%s' "$output"
}
probe_public_health() {
local phase="$1"
local http_code
local status
set +e
http_code=$(curl -sS -w "%{http_code}" -o /dev/null --connect-timeout 3 --max-time 8 "${{ env.API_HEALTH_URL }}" 2>/dev/null)
status=$?
set -e
if [ "$status" -ne 0 ]; then
http_code="curl_error_${status}"
fi
if [ "$http_code" != "200" ]; then
HEALTH_FAILURE_COUNT=$((HEALTH_FAILURE_COUNT + 1))
if [ "$HEALTH_FAILURE_COUNT" -le 3 ]; then
record_rollout_risk "public_health_${phase}_http=${http_code}"
fi
fi
}
collect_argocd_resource_evidence() {
local template
local output
local status
template='{{range .status.resources}}{{if ne .status "Synced"}}{{.kind}}/{{.name}}{{if .namespace}} ns={{.namespace}}{{end}} sync={{.status}}{{if .health.status}} health={{.health.status}}{{end}}{{"\n"}}{{end}}{{if .health.status}}{{if ne .health.status "Healthy"}}{{.kind}}/{{.name}}{{if .namespace}} ns={{.namespace}}{{end}} sync={{.status}} health={{.health.status}}{{if .health.message}} msg={{.health.message}}{{end}}{{"\n"}}{{end}}{{end}}{{end}}'
set +e
output=$($KUBECTL get application awoooi-prod -n argocd -o "go-template=${template}" 2>&1)
status=$?
set -e
if [ "$status" -ne 0 ]; then
local output_snippet
output_snippet=$(printf '%s' "$output" | head -c 180)
echo "resource_query_failed=${output_snippet}"
return 0
fi
echo "$output" \
| awk 'NF && !seen[$0]++ {print}' \
| head -5 \
| tr '\n' ';' \
| sed 's/[[:cntrl:]]//g; s/;*$//'
}
validate_argocd_source_contract() {
local target_revision
local image_override
target_revision=$(app_field '{.spec.source.targetRevision}' source_target_revision)
image_override=$(app_field '{.spec.source.kustomize.images}' source_kustomize_images)
if [ "$target_revision" != "main" ]; then
record_rollout_risk "argocd_source_target_revision_not_main targetRevision=$target_revision"
echo "❌ ArgoCD source targetRevision must be main, got: $target_revision" >&2
exit 1
fi
if [ -n "$image_override" ]; then
local image_override_snippet
image_override_snippet=$(printf '%s' "$image_override" | head -c 180)
record_rollout_risk "argocd_source_image_override_present images=${image_override_snippet}"
echo "❌ ArgoCD source kustomize.images override must be empty; image truth belongs in k8s/awoooi-prod/kustomization.yaml" >&2
exit 1
fi
}
# 等待 ArgoCD Application 同步到目標 revision（最多 180s）。
# 2026-05-24 Codex: top-level Application health can stay Degraded
# without per-resource health detail. Treat that as rollout evidence,
# then let kubectl rollout status and API health decide pass/fail.
echo "⏳ 等待 ArgoCD sync..."
validate_argocd_source_contract
$KUBECTL annotate application awoooi-prod -n argocd \
argocd.argoproj.io/refresh=hard --overwrite >/dev/null 2>&1 || true
for i in $(seq 1 36); do
SYNC=$(app_field '{.status.sync.status}' sync)
HEALTH=$(app_field '{.status.health.status}' health)
REVISION=$(app_field '{.status.sync.revision}' revision)
SHORT_REVISION=$(echo "$REVISION" | cut -c1-8)
SHORT_EXPECTED=$(echo "$EXPECTED_REVISION" | cut -c1-8)
echo " ArgoCD: sync=$SYNC health=$HEALTH revision=$SHORT_REVISION expected=${SHORT_EXPECTED:-any}"
probe_public_health "argocd_wait"
if [ "$SYNC" = "Unknown" ] || [ "$HEALTH" = "Unknown" ] || [ "$REVISION" = "Unknown" ]; then
UNKNOWN_STATUS_COUNT=$((UNKNOWN_STATUS_COUNT + 1))
if [ "$UNKNOWN_STATUS_COUNT" -le 3 ]; then
record_rollout_risk "argocd_status_unknown sync=$SYNC health=$HEALTH revision=$SHORT_REVISION expected=${SHORT_EXPECTED:-any}"
fi
fi
if [ "$SYNC" = "Synced" ]; then
if [ -z "$EXPECTED_REVISION" ] || [ "$REVISION" = "$EXPECTED_REVISION" ]; then
if [ "$HEALTH" != "Healthy" ]; then
RESOURCE_EVIDENCE=$(collect_argocd_resource_evidence)
if [ -n "$RESOURCE_EVIDENCE" ]; then
record_rollout_risk "argocd_health_not_healthy health=$HEALTH revision=$SHORT_REVISION resources=$RESOURCE_EVIDENCE"
else
record_rollout_risk "argocd_health_not_healthy health=$HEALTH revision=$SHORT_REVISION resources=none_visible"
fi
fi
echo "✅ ArgoCD Synced to target revision (health=$HEALTH)"
break
fi
fi
if [ "$i" = "36" ]; then
echo "❌ ArgoCD 未在期限內同步到目標 revision"
exit 1
fi
sleep 5
done
wait_for_rollout() {
local workload="$1"
DEPLOY_STAGE="rollout_${workload}"
if ! $KUBECTL rollout status "deployment/${workload}" \
-n awoooi-prod --timeout="${ROLLOUT_TIMEOUT_SECONDS}s"; then
record_rollout_risk \
"deployment_rollout_timeout workload=${workload} timeout_seconds=${ROLLOUT_TIMEOUT_SECONDS}"
emit_deploy_terminal_receipt \
"blocked_with_safe_next_action" "$DEPLOY_STAGE" "$workload"
return 1
fi
}
# Two-replica rollouts can legitimately need more than two minutes.
# Keep convergence bounded by the declared probe budget, and leave a
# terminal receipt before the independent boundary verifier can run.
wait_for_rollout awoooi-api || exit 1
wait_for_rollout awoooi-web || exit 1
wait_for_rollout awoooi-worker || exit 1
wait_for_rollout awoooi-ansible-executor-broker || exit 1
echo "✅ 部署完成"
# AIA-P0-011: production must prove that the public API cannot mutate
# workloads or read host executor credentials. Source YAML alone is
# insufficient because stale SSA-owned list fields can survive sync.
auth_can_i() {
local output
local status
local answer
set +e
output=$($KUBECTL auth can-i "$@" 2>&1)
status=$?
set -e
answer=$(printf '%s\n' "$output" | tail -1 | tr -d '\r')
case "$answer" in
yes|no)
printf '%s' "$answer"
return 0
;;
*)
echo "❌ kubectl auth can-i returned an invalid response: $output" >&2
if [ "$status" -eq 0 ]; then
status=1
fi
return "$status"
;;
esac
}
DEPLOY_STAGE="executor_boundary_collect_live_identity_and_policy"
echo "executor_boundary_stage=collect_live_identity_and_policy"
API_SA=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \
-o jsonpath='{.spec.template.spec.serviceAccountName}')
WORKER_SA=$($KUBECTL get deployment/awoooi-worker -n awoooi-prod \
-o jsonpath='{.spec.template.spec.serviceAccountName}')
BROKER_SA=$($KUBECTL get deployment/awoooi-ansible-executor-broker -n awoooi-prod \
-o jsonpath='{.spec.template.spec.serviceAccountName}')
WORKER_MOUNTS=$($KUBECTL get deployment/awoooi-worker -n awoooi-prod \
-o jsonpath='{range .spec.template.spec.containers[?(@.name=="worker")].volumeMounts[*]}{.name}{"\n"}{end}')
WORKER_VOLUMES=$($KUBECTL get deployment/awoooi-worker -n awoooi-prod \
-o jsonpath='{range .spec.template.spec.volumes[*]}{.name}{"\n"}{end}')
BROKER_AUTOMOUNT=$($KUBECTL get deployment/awoooi-ansible-executor-broker -n awoooi-prod \
-o jsonpath='{.spec.template.spec.automountServiceAccountToken}')
BROKER_MOUNTS=$($KUBECTL get deployment/awoooi-ansible-executor-broker -n awoooi-prod \
-o jsonpath='{range .spec.template.spec.containers[?(@.name=="broker")].volumeMounts[*]}{.name}{"\n"}{end}')
API_MOUNTS=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \
-o jsonpath='{range .spec.template.spec.containers[?(@.name=="api")].volumeMounts[*]}{.name}{"\n"}{end}')
API_VOLUMES=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \
-o jsonpath='{range .spec.template.spec.volumes[*]}{.name}{"\n"}{end}')
API_FS_GROUP=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \
-o jsonpath='{.spec.template.spec.securityContext.fsGroup}')
API_SSH_MCP=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].env[?(@.name=="SSH_MCP_ENABLED")].value}')
API_CHECK_WORKER=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].env[?(@.name=="ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER")].value}')
API_BACKFILL_WORKER=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].env[?(@.name=="ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER")].value}')
API_CAN_GET=$(auth_can_i \
--as=system:serviceaccount:awoooi-prod:awoooi-api \
get pods --all-namespaces)
API_CAN_PATCH=$(auth_can_i \
--as=system:serviceaccount:awoooi-prod:awoooi-api \
patch deployments.apps --all-namespaces)
API_CAN_DELETE=$(auth_can_i \
--as=system:serviceaccount:awoooi-prod:awoooi-api \
delete pods --all-namespaces)
WORKER_CAN_PATCH=$(auth_can_i \
--as=system:serviceaccount:awoooi-prod:awoooi-worker \
patch deployments.apps --all-namespaces)
WORKER_CAN_DELETE=$(auth_can_i \
--as=system:serviceaccount:awoooi-prod:awoooi-worker \
delete pods --all-namespaces)
BROKER_CAN_PATCH=$(auth_can_i \
--as=system:serviceaccount:awoooi-prod:awoooi-executor-broker \
patch deployments.apps --all-namespaces)
BROKER_CAN_DELETE=$(auth_can_i \
--as=system:serviceaccount:awoooi-prod:awoooi-executor-broker \
delete pods --all-namespaces)
COMMON_SSH_PORT_COUNT=$($KUBECTL get networkpolicy \
allow-required-egress -n awoooi-prod \
-o jsonpath='{range .spec.egress[*].ports[*]}{.port}{"\n"}{end}' \
| grep -c '^22$' || true)
BROKER_SSH_SELECTOR=$($KUBECTL get networkpolicy \
allow-ansible-executor-broker-ssh-egress -n awoooi-prod \
-o jsonpath='{.spec.podSelector.matchLabels.app}')
BROKER_SSH_COMPONENT=$($KUBECTL get networkpolicy \
allow-ansible-executor-broker-ssh-egress -n awoooi-prod \
-o jsonpath='{.spec.podSelector.matchLabels.component}')
BROKER_SSH_PORTS=$($KUBECTL get networkpolicy \
allow-ansible-executor-broker-ssh-egress -n awoooi-prod \
-o jsonpath='{range .spec.egress[*].ports[*]}{.port}{"\n"}{end}')
BROKER_SSH_CIDRS=$($KUBECTL get networkpolicy \
allow-ansible-executor-broker-ssh-egress -n awoooi-prod \
-o jsonpath='{range .spec.egress[*].to[*]}{.ipBlock.cidr}{"\n"}{end}')
LEGACY_EGRESS_RULE_COUNT=$($KUBECTL get networkpolicy \
deny-legacy-access -n awoooi-prod -o json \
| python3 -c 'import json,sys; print(len(json.load(sys.stdin)["spec"].get("egress") or []))')
test "$API_SA" = "awoooi-api" || {
echo "❌ API ServiceAccount drift: $API_SA"; exit 1;
}
test "$WORKER_SA" = "awoooi-worker" || {
echo "❌ worker ServiceAccount drift: $WORKER_SA"; exit 1;
}
test "$BROKER_SA" = "awoooi-executor-broker" || {
echo "❌ broker ServiceAccount drift: $BROKER_SA"; exit 1;
}
if printf '%s\n%s\n' "$API_MOUNTS" "$API_VOLUMES" \
| grep -Eq '^(repair-ssh-key|repair-known-hosts|ssh-mcp-key)$'; then
echo "❌ API still mounts executor SSH credentials"
exit 1
fi
test -z "$API_FS_GROUP" || {
echo "❌ API still carries executor fsGroup: $API_FS_GROUP"; exit 1;
}
test "$API_SSH_MCP" = "false" || {
echo "❌ API SSH MCP is not disabled"; exit 1;
}
test "$API_CHECK_WORKER" = "false" || {
echo "❌ API check-mode executor loop is not disabled"; exit 1;
}
test "$API_BACKFILL_WORKER" = "false" || {
echo "❌ API candidate backfill executor loop is not disabled"; exit 1;
}
test "$API_CAN_GET" = "yes" || {
echo "❌ API read capability missing"; exit 1;
}
test "$API_CAN_PATCH" = "no" || {
echo "❌ API can still patch workloads"; exit 1;
}
test "$API_CAN_DELETE" = "no" || {
echo "❌ API can still delete pods"; exit 1;
}
if printf '%s\n%s\n' "$WORKER_MOUNTS" "$WORKER_VOLUMES" \
| grep -Eq '^(repair-ssh-key|repair-known-hosts|ssh-mcp-key)$'; then
echo "❌ signal worker still mounts executor SSH credentials"
exit 1
fi
test "$BROKER_AUTOMOUNT" = "false" || {
echo "❌ broker Kubernetes token automount is not disabled"; exit 1;
}
printf '%s\n' "$BROKER_MOUNTS" | grep -qx 'ssh-mcp-key' || {
echo "❌ broker bounded SSH transport mount missing"; exit 1;
}
test "$WORKER_CAN_PATCH" = "no" || {
echo "❌ signal worker can patch workloads"; exit 1;
}
test "$WORKER_CAN_DELETE" = "no" || {
echo "❌ signal worker can delete pods"; exit 1;
}
test "$BROKER_CAN_PATCH" = "no" || {
echo "❌ broker can patch Kubernetes workloads"; exit 1;
}
test "$BROKER_CAN_DELETE" = "no" || {
echo "❌ broker can delete Kubernetes pods"; exit 1;
}
if $KUBECTL get clusterrolebinding awoooi-executor-binding >/dev/null 2>&1; then
echo "❌ legacy cluster-wide executor binding still exists"
exit 1
fi
test "$COMMON_SSH_PORT_COUNT" = "0" || {
echo "❌ common workload egress still grants SSH"; exit 1;
}
test "$BROKER_SSH_SELECTOR" = "awoooi-ansible-executor-broker" || {
echo "❌ broker SSH policy selector drift: $BROKER_SSH_SELECTOR"; exit 1;
}
test "$BROKER_SSH_COMPONENT" = "execution-broker" || {
echo "❌ broker SSH component selector drift: $BROKER_SSH_COMPONENT"; exit 1;
}
test "$BROKER_SSH_PORTS" = "22" || {
echo "❌ broker SSH policy contains unexpected ports: $BROKER_SSH_PORTS"; exit 1;
}
for expected_cidr in \
192.168.0.110/32 192.168.0.112/32 192.168.0.120/32 \
192.168.0.121/32 192.168.0.188/32; do
printf '%s\n' "$BROKER_SSH_CIDRS" | grep -qx "$expected_cidr" || {
echo "❌ broker SSH allowlist missing $expected_cidr"; exit 1;
}
done
test "$(printf '%s\n' "$BROKER_SSH_CIDRS" | sed '/^$/d' | wc -l)" = "5" || {
echo "❌ broker SSH allowlist has unexpected destinations"; exit 1;
}
test "$LEGACY_EGRESS_RULE_COUNT" = "0" || {
echo "❌ legacy namespace policy still creates an allow rule"; exit 1;
}
DEPLOY_STAGE="executor_boundary_static_identity_and_policy_verified"
echo "executor_boundary_stage=static_identity_and_policy_verified"
verify_ssh_denied() {
workload="$1"
container="$2"
$KUBECTL exec -i "deployment/$workload" -n awoooi-prod \
-c "$container" -- python - \
192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 <<'PY'
import socket
import sys
reachable = []
for host in sys.argv[1:]:
try:
connection = socket.create_connection((host, 22), timeout=0.75)
except OSError:
continue
else:
connection.close()
reachable.append(host)
print("unexpected_ssh_reachable=" + ",".join(reachable))
raise SystemExit(1 if reachable else 0)
PY
}
verify_broker_ssh_allowed() {
$KUBECTL exec -i deployment/awoooi-ansible-executor-broker \
-n awoooi-prod -c broker -- python - \
192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 <<'PY'
import socket
import sys
connected = []
refused = []
unavailable = []
for host in sys.argv[1:]:
try:
connection = socket.create_connection((host, 22), timeout=1.0)
except ConnectionRefusedError:
# A TCP refusal proves that the packet crossed the egress
# NetworkPolicy boundary and the target replied. Keep host
# service readiness separate from this security-boundary
# verifier; a stopped sshd must not make a healthy rollout
# look like a policy failure. This is bounded to the fixed
# targets whose exact CIDRs were verified immediately above.
refused.append(host)
except OSError:
# Endpoint availability is a separate runtime signal. The
# exact five-destination NetworkPolicy allowlist above is
# the source of truth for the security boundary; one down
# host must not roll back all healthy workloads.
unavailable.append(host)
else:
connection.close()
connected.append(host)
print("broker_ssh_connected=" + ",".join(connected))
print(
"broker_ssh_refused_but_egress_permitted="
+ ",".join(refused)
)
print(
"broker_ssh_allowlisted_endpoint_unavailable="
+ ",".join(unavailable)
)
# Static verification already proves the exact five-destination
# allowlist. The live probe only needs one positive route signal;
# requiring every host to answer confuses host/service availability
# (for example a booted host with sshd down) with NetworkPolicy
# correctness. Arbitrary OSError values remain non-evidence, so an
# all-timeout/unreachable result still fails closed.
raise SystemExit(0 if connected or refused else 1)
PY
}
verify_ssh_denied awoooi-api api || {
echo "❌ API can still establish host SSH connections"; exit 1;
}
verify_ssh_denied awoooi-worker worker || {
echo "❌ signal worker can still establish host SSH connections"; exit 1;
}
verify_broker_ssh_allowed || {
echo "❌ execution broker has no live route evidence for allowlisted SSH endpoints"; exit 1;
}
DEPLOY_STAGE="executor_boundary_live_socket_boundary_verified"
echo "executor_boundary_stage=live_socket_boundary_verified"
read_workload_db_identity_once() {
workload="$1"
container="$2"
representative_probe_count="$3"
required_connection_budget="$4"
last_failure="workload_not_ready"
ready_pod_ref=""
if ! $KUBECTL rollout status "deployment/$workload" \
-n awoooi-prod --timeout=20s >/dev/null 2>&1; then
last_failure="deployment_rollout_not_ready"
else
ready_pod_ref=$(
$KUBECTL get pods -n awoooi-prod -l "app=$workload" \
--field-selector=status.phase=Running \
--sort-by=.metadata.creationTimestamp \
-o name 2>/dev/null | tail -n 1
) || ready_pod_ref=""
if [ -z "$ready_pod_ref" ]; then
last_failure="running_pod_unavailable"
elif ! $KUBECTL wait --for=condition=Ready "$ready_pod_ref" \
-n awoooi-prod --timeout=10s >/dev/null 2>&1; then
last_failure="ready_pod_unavailable"
else
container_ready=$(
$KUBECTL get "$ready_pod_ref" -n awoooi-prod \
-o jsonpath="{.status.containerStatuses[?(@.name=='$container')].ready}" \
2>/dev/null
) || container_ready=""
if [ "$container_ready" != "true" ]; then
last_failure="container_not_ready"
elif identity_output=$(
$KUBECTL exec -i "$ready_pod_ref" -n awoooi-prod \
-c "$container" -- env \
AWOOOI_DB_REPRESENTATIVE_PROBE_COUNT="$representative_probe_count" \
AWOOOI_DB_REQUIRED_CONNECTION_BUDGET="$required_connection_budget" \
python - <<'PY'
import asyncio
import hashlib
import json
import os
from sqlalchemy import text
from src.db.base import get_db_context
async def main() -> None:
try:
async with get_db_context("awoooi") as db:
result = await db.execute(text("""
SELECT
current_user AS role_name,
rolconnlimit AS connection_limit,
(
SELECT count(*)
FROM pg_stat_activity
WHERE usename = current_user
AND pid <> pg_backend_pid()
) AS active_role_connection_count
FROM pg_roles
WHERE rolname = current_user
"""))
row = result.mappings().one()
for table_name in (
"incidents",
"knowledge_entries",
"awooop_run_state",
):
await db.execute(
text(f'SELECT count(*) FROM "{table_name}"')
)
table_gap_count = int((await db.execute(text("""
SELECT count(*)
FROM pg_class relation
JOIN pg_namespace namespace
ON namespace.oid = relation.relnamespace
WHERE namespace.nspname = 'public'
AND relation.relkind IN ('r', 'p', 'v', 'm')
AND (
(has_table_privilege('awoooi', relation.oid, 'SELECT')
AND NOT has_table_privilege(current_user, relation.oid, 'SELECT'))
OR (has_table_privilege('awoooi', relation.oid, 'INSERT')
AND NOT has_table_privilege(current_user, relation.oid, 'INSERT'))
OR (has_table_privilege('awoooi', relation.oid, 'UPDATE')
AND NOT has_table_privilege(current_user, relation.oid, 'UPDATE'))
OR (has_table_privilege('awoooi', relation.oid, 'DELETE')
AND NOT has_table_privilege(current_user, relation.oid, 'DELETE'))
)
"""))).scalar_one())
sequence_gap_count = int((await db.execute(text("""
SELECT count(*)
FROM pg_class sequence
JOIN pg_namespace namespace
ON namespace.oid = sequence.relnamespace
WHERE namespace.nspname = 'public'
AND sequence.relkind = 'S'
AND (
(has_sequence_privilege('awoooi', sequence.oid, 'USAGE')
AND NOT has_sequence_privilege(current_user, sequence.oid, 'USAGE'))
OR (has_sequence_privilege('awoooi', sequence.oid, 'SELECT')
AND NOT has_sequence_privilege(current_user, sequence.oid, 'SELECT'))
OR (has_sequence_privilege('awoooi', sequence.oid, 'UPDATE')
AND NOT has_sequence_privilege(current_user, sequence.oid, 'UPDATE'))
)
"""))).scalar_one())
representative_probe_count = int(
os.environ["AWOOOI_DB_REPRESENTATIVE_PROBE_COUNT"]
)
required_connection_budget = int(
os.environ["AWOOOI_DB_REQUIRED_CONNECTION_BUDGET"]
)
if representative_probe_count < 1:
raise RuntimeError("representative_probe_count_invalid")
if required_connection_budget < 1:
raise RuntimeError("required_connection_budget_invalid")
connection_limit = int(row["connection_limit"])
active_role_connection_count = int(
row["active_role_connection_count"]
)
available_connection_headroom = (
connection_limit - active_role_connection_count
)
async def probe_connection() -> None:
async with get_db_context("awoooi") as probe_db:
await probe_db.execute(text("SELECT pg_sleep(0.25)"))
await asyncio.gather(*(
probe_connection()
for _ in range(representative_probe_count)
))
except Exception as exc:
print(json.dumps({
"role_fingerprint": "",
"connection_limit": None,
"required_connection_budget": None,
"active_role_connection_count": None,
"available_connection_headroom": None,
"representative_connection_probe_count": 0,
"representative_preflight_passed": False,
"table_privilege_gap_count": None,
"sequence_privilege_gap_count": None,
"error_type": type(exc).__name__,
}))
return
print(json.dumps({
"role_fingerprint": hashlib.sha256(
str(row["role_name"]).encode("utf-8")
).hexdigest(),
"connection_limit": connection_limit,
"required_connection_budget": required_connection_budget,
"active_role_connection_count": active_role_connection_count,
"available_connection_headroom": available_connection_headroom,
"representative_connection_probe_count": representative_probe_count,
"representative_preflight_passed": (
table_gap_count == 0
and sequence_gap_count == 0
and representative_probe_count >= 1
and connection_limit >= required_connection_budget
and (
available_connection_headroom
>= required_connection_budget
)
),
"table_privilege_gap_count": table_gap_count,
"sequence_privilege_gap_count": sequence_gap_count,
"error_type": None,
}))
asyncio.run(main())
PY
); then
identity_json=$(printf '%s\n' "$identity_output" | tail -n 1)
if printf '%s' "$identity_json" | python3 -c '
import json
import sys
payload = json.load(sys.stdin)
valid = (
bool(payload.get("role_fingerprint"))
and payload.get("representative_preflight_passed") is True
and int(
payload.get("representative_connection_probe_count") or 0
) >= 1
)
raise SystemExit(0 if valid else 1)
' >/dev/null 2>&1; then
printf '%s\n' "$identity_json"
return 0
fi
last_failure="identity_preflight_not_ready"
else
last_failure="kubectl_exec_failed"
fi
fi
fi
echo "workload_db_identity_once_failed workload=$workload container=$container reason=$last_failure" >&2
return 1
}
read_workload_db_identity() {
workload="$1"
retry_backoff_seconds="${AWOOOI_DB_IDENTITY_RETRY_BACKOFF_SECONDS:-5}"
case "$retry_backoff_seconds" in
''|*[!0-9]*) retry_backoff_seconds=5 ;;
esac
attempt=1
while [ "$attempt" -le 3 ]; do
if identity_output=$(read_workload_db_identity_once "$@"); then
printf '%s\n' "$identity_output"
return 0
fi
echo "workload_db_identity_exec_retry=$attempt/3;workload=$workload" >&2
if [ "$attempt" -lt 3 ]; then
sleep "$retry_backoff_seconds"
fi
attempt=$((attempt + 1))
done
echo "workload_db_identity_exec_exhausted=3;workload=$workload" >&2
printf '%s\n' '{"role_fingerprint":"","connection_limit":null,"required_connection_budget":null,"active_role_connection_count":null,"available_connection_headroom":null,"representative_connection_probe_count":0,"representative_preflight_passed":false,"table_privilege_gap_count":null,"sequence_privilege_gap_count":null,"error_type":"WorkloadDbIdentityAttemptsExhausted"}'
return 1
}
json_field() {
field="$1"
python3 -c \
'import json,sys; value=json.load(sys.stdin).get(sys.argv[1]); print(value if value is not None else "")' \
"$field"
}
normalize_bool() {
case "${1:-}" in
true|TRUE|True|1) printf 'true' ;;
*) printf 'false' ;;
esac
}
workload_secret_projection() {
workload="$1"
container="$2"
explicit_ref=$($KUBECTL get "deployment/$workload" -n awoooi-prod \
-o jsonpath="{.spec.template.spec.containers[?(@.name=='$container')].env[?(@.name=='DATABASE_URL')].valueFrom.secretKeyRef.name}" \
|| true)
env_from_refs=$($KUBECTL get "deployment/$workload" -n awoooi-prod \
-o jsonpath="{range .spec.template.spec.containers[?(@.name=='$container')].envFrom[*]}{.secretRef.name}{'\\n'}{end}" \
| sed '/^$/d' | paste -sd, - || true)
if [ -n "$explicit_ref" ]; then
printf '%s|%s\n' "$explicit_ref" "$env_from_refs"
else
printf '%s|%s\n' "$env_from_refs" "$env_from_refs"
fi
}
if API_DB_IDENTITY=$(read_workload_db_identity awoooi-api api 1 8); then
API_DB_IDENTITY_EXIT=0
else
API_DB_IDENTITY_EXIT=$?
fi
if WORKER_DB_IDENTITY=$(read_workload_db_identity awoooi-worker worker 1 5); then
WORKER_DB_IDENTITY_EXIT=0
else
WORKER_DB_IDENTITY_EXIT=$?
fi
if BROKER_DB_IDENTITY=$(read_workload_db_identity \
awoooi-ansible-executor-broker broker 1 2); then
BROKER_DB_IDENTITY_EXIT=0
else
BROKER_DB_IDENTITY_EXIT=$?
fi
if [ "$API_DB_IDENTITY_EXIT" -ne 0 ] \
|| [ "$WORKER_DB_IDENTITY_EXIT" -ne 0 ] \
|| [ "$BROKER_DB_IDENTITY_EXIT" -ne 0 ]; then
echo "❌ workload DB identity attempts exhausted before receipt write"
exit 1
fi
API_DB_ROLE_FINGERPRINT=$(printf '%s' "$API_DB_IDENTITY" \
| json_field role_fingerprint 2>/dev/null || true)
WORKER_DB_ROLE_FINGERPRINT=$(printf '%s' "$WORKER_DB_IDENTITY" \
| json_field role_fingerprint 2>/dev/null || true)
BROKER_DB_ROLE_FINGERPRINT=$(printf '%s' "$BROKER_DB_IDENTITY" \
| json_field role_fingerprint 2>/dev/null || true)
API_DB_CONNECTION_LIMIT=$(printf '%s' "$API_DB_IDENTITY" \
| json_field connection_limit 2>/dev/null || true)
WORKER_DB_CONNECTION_LIMIT=$(printf '%s' "$WORKER_DB_IDENTITY" \
| json_field connection_limit 2>/dev/null || true)
BROKER_DB_CONNECTION_LIMIT=$(printf '%s' "$BROKER_DB_IDENTITY" \
| json_field connection_limit 2>/dev/null || true)
API_DB_REQUIRED_CONNECTION_BUDGET=$(printf '%s' "$API_DB_IDENTITY" \
| json_field required_connection_budget 2>/dev/null || true)
WORKER_DB_REQUIRED_CONNECTION_BUDGET=$(printf '%s' "$WORKER_DB_IDENTITY" \
| json_field required_connection_budget 2>/dev/null || true)
BROKER_DB_REQUIRED_CONNECTION_BUDGET=$(printf '%s' "$BROKER_DB_IDENTITY" \
| json_field required_connection_budget 2>/dev/null || true)
API_DB_ACTIVE_ROLE_CONNECTION_COUNT=$(printf '%s' "$API_DB_IDENTITY" \
| json_field active_role_connection_count 2>/dev/null || true)
WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT=$(printf '%s' "$WORKER_DB_IDENTITY" \
| json_field active_role_connection_count 2>/dev/null || true)
BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT=$(printf '%s' "$BROKER_DB_IDENTITY" \
| json_field active_role_connection_count 2>/dev/null || true)
API_DB_AVAILABLE_CONNECTION_HEADROOM=$(printf '%s' "$API_DB_IDENTITY" \
| json_field available_connection_headroom 2>/dev/null || true)
WORKER_DB_AVAILABLE_CONNECTION_HEADROOM=$(printf '%s' "$WORKER_DB_IDENTITY" \
| json_field available_connection_headroom 2>/dev/null || true)
BROKER_DB_AVAILABLE_CONNECTION_HEADROOM=$(printf '%s' "$BROKER_DB_IDENTITY" \
| json_field available_connection_headroom 2>/dev/null || true)
API_DB_REPRESENTATIVE_PROBE_COUNT=$(printf '%s' "$API_DB_IDENTITY" \
| json_field representative_connection_probe_count 2>/dev/null || true)
WORKER_DB_REPRESENTATIVE_PROBE_COUNT=$(printf '%s' "$WORKER_DB_IDENTITY" \
| json_field representative_connection_probe_count 2>/dev/null || true)
BROKER_DB_REPRESENTATIVE_PROBE_COUNT=$(printf '%s' "$BROKER_DB_IDENTITY" \
| json_field representative_connection_probe_count 2>/dev/null || true)
API_DB_REPRESENTATIVE_PREFLIGHT=$(normalize_bool "$(printf '%s' "$API_DB_IDENTITY" \
| json_field representative_preflight_passed 2>/dev/null || true)")
WORKER_DB_REPRESENTATIVE_PREFLIGHT=$(normalize_bool "$(printf '%s' "$WORKER_DB_IDENTITY" \
| json_field representative_preflight_passed 2>/dev/null || true)")
BROKER_DB_REPRESENTATIVE_PREFLIGHT=$(normalize_bool "$(printf '%s' "$BROKER_DB_IDENTITY" \
| json_field representative_preflight_passed 2>/dev/null || true)")
API_DB_TABLE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$API_DB_IDENTITY" \
| json_field table_privilege_gap_count 2>/dev/null || true)
WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$WORKER_DB_IDENTITY" \
| json_field table_privilege_gap_count 2>/dev/null || true)
BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$BROKER_DB_IDENTITY" \
| json_field table_privilege_gap_count 2>/dev/null || true)
API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$API_DB_IDENTITY" \
| json_field sequence_privilege_gap_count 2>/dev/null || true)
WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$WORKER_DB_IDENTITY" \
| json_field sequence_privilege_gap_count 2>/dev/null || true)
BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$BROKER_DB_IDENTITY" \
| json_field sequence_privilege_gap_count 2>/dev/null || true)
API_DB_CONNECTION_LIMIT=${API_DB_CONNECTION_LIMIT:-0}
WORKER_DB_CONNECTION_LIMIT=${WORKER_DB_CONNECTION_LIMIT:-0}
BROKER_DB_CONNECTION_LIMIT=${BROKER_DB_CONNECTION_LIMIT:-0}
API_DB_REQUIRED_CONNECTION_BUDGET=${API_DB_REQUIRED_CONNECTION_BUDGET:-0}
WORKER_DB_REQUIRED_CONNECTION_BUDGET=${WORKER_DB_REQUIRED_CONNECTION_BUDGET:-0}
BROKER_DB_REQUIRED_CONNECTION_BUDGET=${BROKER_DB_REQUIRED_CONNECTION_BUDGET:-0}
API_DB_ACTIVE_ROLE_CONNECTION_COUNT=${API_DB_ACTIVE_ROLE_CONNECTION_COUNT:--1}
WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT=${WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT:--1}
BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT=${BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT:--1}
API_DB_AVAILABLE_CONNECTION_HEADROOM=${API_DB_AVAILABLE_CONNECTION_HEADROOM:--1}
WORKER_DB_AVAILABLE_CONNECTION_HEADROOM=${WORKER_DB_AVAILABLE_CONNECTION_HEADROOM:--1}
BROKER_DB_AVAILABLE_CONNECTION_HEADROOM=${BROKER_DB_AVAILABLE_CONNECTION_HEADROOM:--1}
API_DB_REPRESENTATIVE_PROBE_COUNT=${API_DB_REPRESENTATIVE_PROBE_COUNT:-0}
WORKER_DB_REPRESENTATIVE_PROBE_COUNT=${WORKER_DB_REPRESENTATIVE_PROBE_COUNT:-0}
BROKER_DB_REPRESENTATIVE_PROBE_COUNT=${BROKER_DB_REPRESENTATIVE_PROBE_COUNT:-0}
API_DB_TABLE_PRIVILEGE_GAP_COUNT=${API_DB_TABLE_PRIVILEGE_GAP_COUNT:-1}
WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT=${WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT:-1}
BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT=${BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT:-1}
API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=${API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT:-1}
WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=${WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT:-1}
BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=${BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT:-1}
API_DB_PROJECTION=$(workload_secret_projection awoooi-api api)
WORKER_DB_PROJECTION=$(workload_secret_projection awoooi-worker worker)
BROKER_DB_PROJECTION=$(workload_secret_projection \
awoooi-ansible-executor-broker broker)
API_DATABASE_SECRET_REF=${API_DB_PROJECTION%%|*}
WORKER_DATABASE_SECRET_REF=${WORKER_DB_PROJECTION%%|*}
BROKER_DATABASE_SECRET_REF=${BROKER_DB_PROJECTION%%|*}
API_SECRET_ENVFROM_REFS=${API_DB_PROJECTION#*|}
WORKER_SECRET_ENVFROM_REFS=${WORKER_DB_PROJECTION#*|}
BROKER_SECRET_ENVFROM_REFS=${BROKER_DB_PROJECTION#*|}
API_MONOLITHIC_SECRET_ENVFROM_ABSENT=false
WORKER_MONOLITHIC_SECRET_ENVFROM_ABSENT=false
BROKER_MONOLITHIC_SECRET_ENVFROM_ABSENT=false
if [ -z "$API_SECRET_ENVFROM_REFS" ]; then
API_MONOLITHIC_SECRET_ENVFROM_ABSENT=true
fi
if [ -z "$WORKER_SECRET_ENVFROM_REFS" ]; then
WORKER_MONOLITHIC_SECRET_ENVFROM_ABSENT=true
fi
if [ -z "$BROKER_SECRET_ENVFROM_REFS" ]; then
BROKER_MONOLITHIC_SECRET_ENVFROM_ABSENT=true
fi
API_DB_NULL_POOL=$(normalize_bool "$($KUBECTL get deployment/awoooi-api \
-n awoooi-prod -o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].env[?(@.name=="DATABASE_NULL_POOL")].value}')")
WORKER_DB_NULL_POOL=$(normalize_bool "$($KUBECTL get deployment/awoooi-worker \
-n awoooi-prod -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="DATABASE_NULL_POOL")].value}')")
BROKER_DB_NULL_POOL=$(normalize_bool "$($KUBECTL get deployment/awoooi-ansible-executor-broker \
-n awoooi-prod -o jsonpath='{.spec.template.spec.containers[?(@.name=="broker")].env[?(@.name=="DATABASE_NULL_POOL")].value}')")
DISTINCT_DATABASE_SECRET_REFS=false
if [ -n "$API_DATABASE_SECRET_REF" ] \
&& [ -n "$WORKER_DATABASE_SECRET_REF" ] \
&& [ -n "$BROKER_DATABASE_SECRET_REF" ] \
&& [ "$API_DATABASE_SECRET_REF" != "$WORKER_DATABASE_SECRET_REF" ] \
&& [ "$API_DATABASE_SECRET_REF" != "$BROKER_DATABASE_SECRET_REF" ] \
&& [ "$WORKER_DATABASE_SECRET_REF" != "$BROKER_DATABASE_SECRET_REF" ]; then
DISTINCT_DATABASE_SECRET_REFS=true
fi
DISTINCT_DB_ROLE_FINGERPRINTS=false
if [ -n "$API_DB_ROLE_FINGERPRINT" ] \
&& [ -n "$WORKER_DB_ROLE_FINGERPRINT" ] \
&& [ -n "$BROKER_DB_ROLE_FINGERPRINT" ] \
&& [ "$API_DB_ROLE_FINGERPRINT" != "$WORKER_DB_ROLE_FINGERPRINT" ] \
&& [ "$API_DB_ROLE_FINGERPRINT" != "$BROKER_DB_ROLE_FINGERPRINT" ] \
&& [ "$WORKER_DB_ROLE_FINGERPRINT" != "$BROKER_DB_ROLE_FINGERPRINT" ]; then
DISTINCT_DB_ROLE_FINGERPRINTS=true
fi
DB_IDENTITY_ISOLATED=false
if [ "$API_MONOLITHIC_SECRET_ENVFROM_ABSENT" = "true" ] \
&& [ "$WORKER_MONOLITHIC_SECRET_ENVFROM_ABSENT" = "true" ] \
&& [ "$BROKER_MONOLITHIC_SECRET_ENVFROM_ABSENT" = "true" ] \
&& [ "$DISTINCT_DATABASE_SECRET_REFS" = "true" ] \
&& [ "$DISTINCT_DB_ROLE_FINGERPRINTS" = "true" ]; then
DB_IDENTITY_ISOLATED=true
fi
REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED=false
if [ "$API_DB_REPRESENTATIVE_PREFLIGHT" = "true" ] \
&& [ "$WORKER_DB_REPRESENTATIVE_PREFLIGHT" = "true" ] \
&& [ "$BROKER_DB_REPRESENTATIVE_PREFLIGHT" = "true" ] \
&& [ "$API_DB_REPRESENTATIVE_PROBE_COUNT" -ge 1 ] \
&& [ "$WORKER_DB_REPRESENTATIVE_PROBE_COUNT" -ge 1 ] \
&& [ "$BROKER_DB_REPRESENTATIVE_PROBE_COUNT" -ge 1 ] \
&& [ "$API_DB_TABLE_PRIVILEGE_GAP_COUNT" -eq 0 ] \
&& [ "$WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT" -eq 0 ] \
&& [ "$BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT" -eq 0 ] \
&& [ "$API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" -eq 0 ] \
&& [ "$WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" -eq 0 ] \
&& [ "$BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" -eq 0 ]; then
REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED=true
fi
CONNECTION_BUDGET_VERIFIED=false
if [ "$DB_IDENTITY_ISOLATED" = "true" ] \
&& [ "$REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED" = "true" ] \
&& [ "$API_DB_NULL_POOL" = "true" ] \
&& [ "$WORKER_DB_NULL_POOL" = "true" ] \
&& [ "$BROKER_DB_NULL_POOL" = "true" ] \
&& [ "$API_DB_CONNECTION_LIMIT" -eq 12 ] \
&& [ "$WORKER_DB_CONNECTION_LIMIT" -eq 8 ] \
&& [ "$BROKER_DB_CONNECTION_LIMIT" -eq 4 ] \
&& [ "$API_DB_REQUIRED_CONNECTION_BUDGET" -eq 8 ] \
&& [ "$WORKER_DB_REQUIRED_CONNECTION_BUDGET" -eq 5 ] \
&& [ "$BROKER_DB_REQUIRED_CONNECTION_BUDGET" -eq 2 ] \
&& [ "$API_DB_ACTIVE_ROLE_CONNECTION_COUNT" -ge 0 ] \
&& [ "$WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" -ge 0 ] \
&& [ "$BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" -ge 0 ] \
&& [ "$API_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$API_DB_REQUIRED_CONNECTION_BUDGET" ] \
&& [ "$WORKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$WORKER_DB_REQUIRED_CONNECTION_BUDGET" ] \
&& [ "$BROKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$BROKER_DB_REQUIRED_CONNECTION_BUDGET" ]; then
CONNECTION_BUDGET_VERIFIED=true
fi
api_restart_total() {
$KUBECTL get pods -n awoooi-prod -l app=awoooi-api -o json \
| python3 -c '
import json
import sys
payload = json.load(sys.stdin)
print(sum(
int(status.get("restartCount") or 0)
for item in payload.get("items", [])
for status in (item.get("status", {}).get("containerStatuses") or [])
if status.get("name") == "api"
))
'
}
API_RESTART_COUNT_BEFORE=$(api_restart_total)
API_DESIRED_REPLICA_COUNT=$($KUBECTL get deployment awoooi-api \
-n awoooi-prod -o jsonpath='{.spec.replicas}')
API_HTTP_CONCURRENCY_VERIFIED=$(python3 - <<'PY'
import concurrent.futures
import os
import sys
import time
import urllib.error
import urllib.request
readiness_url = "https://awoooi.wooo.work/api/v1/health/ready"
db_probe_url = (
"https://awoooi.wooo.work/api/v1/platform/cicd/events"
"?project_id=awoooi&limit=1"
)
transient_http_statuses = {408, 425, 429, 500, 502, 503, 504}
def bounded_int(
name: str,
default: int,
minimum: int,
maximum: int,
) -> int:
try:
value = int(os.environ.get(name, str(default)))
except (TypeError, ValueError):
value = default
return max(minimum, min(value, maximum))
readiness_attempts = bounded_int(
"API_PUBLIC_READINESS_ATTEMPTS", 12, 1, 20
)
readiness_delay_seconds = bounded_int(
"API_PUBLIC_READINESS_DELAY_SECONDS", 5, 0, 30
)
concurrency_attempts = bounded_int(
"API_HTTP_CONCURRENCY_ATTEMPTS", 4, 1, 8
)
concurrency_delay_seconds = bounded_int(
"API_HTTP_CONCURRENCY_DELAY_SECONDS", 5, 0, 30
)
def fetch_status(url: str, timeout: int) -> tuple[int | None, str]:
try:
with urllib.request.urlopen(url, timeout=timeout) as response:
return int(response.status), "response"
except urllib.error.HTTPError as exc:
return int(exc.code), "HTTPError"
except Exception as exc:
return None, type(exc).__name__
readiness_result = "unavailable"
for readiness_attempt in range(1, readiness_attempts + 1):
status, error_type = fetch_status(readiness_url, timeout=10)
if (
status is not None
and status != 200
and status not in transient_http_statuses
):
raise SystemExit(
"API sequential readiness permanent HTTP status: "
+ str(status)
)
readiness_result = (
"status="
+ (str(status) if status is not None else "unavailable")
+ ";error="
+ error_type
)
if status == 200:
print(
"api_sequential_readiness=ready;attempt="
+ str(readiness_attempt),
file=sys.stderr,
)
break
if readiness_attempt < readiness_attempts:
time.sleep(readiness_delay_seconds)
else:
raise SystemExit(
"API sequential readiness failed: " + readiness_result
)
def fetch(url: str) -> int:
status, error_type = fetch_status(url, timeout=20)
if status is None:
raise RuntimeError(error_type)
return status
last_result = "unavailable"
concurrency_urls = (db_probe_url,) * 8
for attempt in range(1, concurrency_attempts + 1):
try:
with concurrent.futures.ThreadPoolExecutor(
max_workers=8
) as executor:
statuses = list(executor.map(fetch, concurrency_urls))
except Exception as exc:
last_result = type(exc).__name__
else:
last_result = ",".join(str(status) for status in statuses)
if all(status == 200 for status in statuses):
print("true")
break
permanent_statuses = sorted({
status
for status in statuses
if status != 200
and status not in transient_http_statuses
})
if permanent_statuses:
raise SystemExit(
"API database concurrency probe permanent HTTP "
"status: "
+ ",".join(
str(status) for status in permanent_statuses
)
)
if attempt < concurrency_attempts:
time.sleep(concurrency_delay_seconds)
else:
print(
"API database concurrency probe degraded: " + last_result,
file=sys.stderr,
)
print("false")
PY
)
if [ "$API_HTTP_CONCURRENCY_VERIFIED" != "true" ]; then
CONNECTION_BUDGET_VERIFIED=false
fi
API_POST_PROBE_ROLLOUT_READY=false
if $KUBECTL rollout status deployment/awoooi-api -n awoooi-prod \
--timeout=120s >/dev/null 2>&1; then
API_POST_PROBE_ROLLOUT_READY=true
fi
API_RESTART_COUNT_AFTER=$(api_restart_total)
API_READY_REPLICA_COUNT=$($KUBECTL get deployment awoooi-api \
-n awoooi-prod -o jsonpath='{.status.readyReplicas}')
API_READY_REPLICA_COUNT=${API_READY_REPLICA_COUNT:-0}
API_PROBE_RESTART_FREE=false
if [ "$API_RESTART_COUNT_AFTER" -eq "$API_RESTART_COUNT_BEFORE" ]; then
API_PROBE_RESTART_FREE=true
fi
if [ "$API_POST_PROBE_ROLLOUT_READY" != "true" ] \
|| [ "$API_PROBE_RESTART_FREE" != "true" ] \
|| [ "$API_READY_REPLICA_COUNT" -ne "$API_DESIRED_REPLICA_COUNT" ]; then
echo "API concurrency probe destabilized rollout: ready=${API_READY_REPLICA_COUNT}/${API_DESIRED_REPLICA_COUNT} restarts=${API_RESTART_COUNT_BEFORE}->${API_RESTART_COUNT_AFTER}" >&2
exit 1
fi
BOUNDARY_VERIFIED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
$KUBECTL create configmap awoooi-executor-boundary-verification \
-n awoooi-prod \
--from-literal=schema_version=awoooi_executor_boundary_verification_v4 \
--from-literal=verified_source_sha="$SOURCE_REVISION" \
--from-literal=verified_at="$BOUNDARY_VERIFIED_AT" \
--from-literal=workflow_run_id="$WORKFLOW_RUN_ID" \
--from-literal=verifier=kubectl_auth_can_i_and_live_mount_readback \
--from-literal=api_service_account="$API_SA" \
--from-literal=worker_service_account="$WORKER_SA" \
--from-literal=broker_service_account="$BROKER_SA" \
--from-literal=api_mutation_denied=true \
--from-literal=api_ssh_mount_absent=true \
--from-literal=worker_mutation_denied=true \
--from-literal=worker_ssh_mount_absent=true \
--from-literal=broker_kubernetes_token_absent=true \
--from-literal=broker_ssh_mount_present=true \
--from-literal=legacy_executor_binding_absent=true \
--from-literal=api_ssh_egress_denied=true \
--from-literal=worker_ssh_egress_denied=true \
--from-literal=broker_ssh_egress_allowlisted=true \
--from-literal=legacy_namespace_egress_allow_absent=true \
--from-literal=single_writer_source_verifier=cd_tests_and_production_source_sha_v1 \
--from-literal=decision_manager_queue_only=true \
--from-literal=webhook_router_queue_only=true \
--from-literal=failure_watcher_queue_only=true \
--from-literal=auto_approved_direct_execution_blocked=true \
--from-literal=candidate_idempotency_contract_verified=true \
--from-literal=apply_idempotency_contract_verified=true \
--from-literal=critical_break_glass_contract_verified=true \
--from-literal=independent_post_verifier_source_verifier=cd_tests_and_production_source_sha_v1 \
--from-literal=asset_postcondition_registry_complete=true \
--from-literal=independent_post_verifier_read_only=true \
--from-literal=executor_returncode_not_success_source=true \
--from-literal=verifier_failure_marks_apply_failed=true \
--from-literal=verifier_failure_retry_first=true \
--from-literal=post_verifier_raw_output_not_persisted=true \
--from-literal=canonical_learning_source_verifier=cd_tests_and_production_source_sha_v1 \
--from-literal=canonical_playbook_resolver_complete=true \
--from-literal=km_row_readback_required=true \
--from-literal=playbook_row_readback_required=true \
--from-literal=learning_failure_fail_closed=true \
--from-literal=trust_failure_fail_closed=true \
--from-literal=learning_receipts_public_safe=true \
--from-literal=db_identity_verifier=runtime_role_fingerprint_role_limit_null_pool_http_concurrency_and_representative_preflight_v4 \
--from-literal=api_database_secret_ref="$API_DATABASE_SECRET_REF" \
--from-literal=worker_database_secret_ref="$WORKER_DATABASE_SECRET_REF" \
--from-literal=broker_database_secret_ref="$BROKER_DATABASE_SECRET_REF" \
--from-literal=api_db_role_fingerprint="$API_DB_ROLE_FINGERPRINT" \
--from-literal=worker_db_role_fingerprint="$WORKER_DB_ROLE_FINGERPRINT" \
--from-literal=broker_db_role_fingerprint="$BROKER_DB_ROLE_FINGERPRINT" \
--from-literal=api_db_connection_limit="$API_DB_CONNECTION_LIMIT" \
--from-literal=worker_db_connection_limit="$WORKER_DB_CONNECTION_LIMIT" \
--from-literal=broker_db_connection_limit="$BROKER_DB_CONNECTION_LIMIT" \
--from-literal=api_required_connection_budget="$API_DB_REQUIRED_CONNECTION_BUDGET" \
--from-literal=worker_required_connection_budget="$WORKER_DB_REQUIRED_CONNECTION_BUDGET" \
--from-literal=broker_required_connection_budget="$BROKER_DB_REQUIRED_CONNECTION_BUDGET" \
--from-literal=api_active_role_connection_count="$API_DB_ACTIVE_ROLE_CONNECTION_COUNT" \
--from-literal=worker_active_role_connection_count="$WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" \
--from-literal=broker_active_role_connection_count="$BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" \
--from-literal=api_available_connection_headroom="$API_DB_AVAILABLE_CONNECTION_HEADROOM" \
--from-literal=worker_available_connection_headroom="$WORKER_DB_AVAILABLE_CONNECTION_HEADROOM" \
--from-literal=broker_available_connection_headroom="$BROKER_DB_AVAILABLE_CONNECTION_HEADROOM" \
--from-literal=api_representative_connection_probe_count="$API_DB_REPRESENTATIVE_PROBE_COUNT" \
--from-literal=worker_representative_connection_probe_count="$WORKER_DB_REPRESENTATIVE_PROBE_COUNT" \
--from-literal=broker_representative_connection_probe_count="$BROKER_DB_REPRESENTATIVE_PROBE_COUNT" \
--from-literal=api_representative_db_preflight_passed="$API_DB_REPRESENTATIVE_PREFLIGHT" \
--from-literal=worker_representative_db_preflight_passed="$WORKER_DB_REPRESENTATIVE_PREFLIGHT" \
--from-literal=broker_representative_db_preflight_passed="$BROKER_DB_REPRESENTATIVE_PREFLIGHT" \
--from-literal=api_table_privilege_gap_count="$API_DB_TABLE_PRIVILEGE_GAP_COUNT" \
--from-literal=worker_table_privilege_gap_count="$WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT" \
--from-literal=broker_table_privilege_gap_count="$BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT" \
--from-literal=api_sequence_privilege_gap_count="$API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \
--from-literal=worker_sequence_privilege_gap_count="$WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \
--from-literal=broker_sequence_privilege_gap_count="$BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \
--from-literal=representative_db_preflights_verified="$REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED" \
--from-literal=api_http_concurrency_probe_passed="$API_HTTP_CONCURRENCY_VERIFIED" \
--from-literal=api_http_probe_restart_free="$API_PROBE_RESTART_FREE" \
--from-literal=api_restart_count_before="$API_RESTART_COUNT_BEFORE" \
--from-literal=api_restart_count_after="$API_RESTART_COUNT_AFTER" \
--from-literal=api_ready_replica_count="$API_READY_REPLICA_COUNT" \
--from-literal=api_desired_replica_count="$API_DESIRED_REPLICA_COUNT" \
--from-literal=api_database_null_pool="$API_DB_NULL_POOL" \
--from-literal=worker_database_null_pool="$WORKER_DB_NULL_POOL" \
--from-literal=broker_database_null_pool="$BROKER_DB_NULL_POOL" \
--from-literal=api_monolithic_secret_env_from_absent="$API_MONOLITHIC_SECRET_ENVFROM_ABSENT" \
--from-literal=worker_monolithic_secret_env_from_absent="$WORKER_MONOLITHIC_SECRET_ENVFROM_ABSENT" \
--from-literal=broker_monolithic_secret_env_from_absent="$BROKER_MONOLITHIC_SECRET_ENVFROM_ABSENT" \
--from-literal=distinct_database_secret_refs="$DISTINCT_DATABASE_SECRET_REFS" \
--from-literal=distinct_db_role_fingerprints="$DISTINCT_DB_ROLE_FINGERPRINTS" \
--from-literal=db_identity_isolated="$DB_IDENTITY_ISOLATED" \
--from-literal=connection_budget_verified="$CONNECTION_BUDGET_VERIFIED" \
--dry-run=client -o yaml \
| $KUBECTL apply -f -
$KUBECTL annotate configmap \
awoooi-executor-boundary-verification -n awoooi-prod \
argocd.argoproj.io/sync-options=Prune=false \
argocd.argoproj.io/compare-options=IgnoreExtraneous \
--overwrite
RECEIPT_SOURCE_SHA=$($KUBECTL get configmap \
awoooi-executor-boundary-verification -n awoooi-prod \
-o jsonpath='{.data.verified_source_sha}')
test "$RECEIPT_SOURCE_SHA" = "$SOURCE_REVISION" || {
echo "❌ executor boundary receipt source mismatch"; exit 1;
}
echo "✅ AIA-P0-011 API/worker/broker production boundary verified"
if [ "$DB_IDENTITY_ISOLATED" != "true" ] \
|| [ "$CONNECTION_BUDGET_VERIFIED" != "true" ]; then
echo "⚠️ AIA-P0-011 workload DB identity/budget remains in progress"
else
echo "✅ AIA-P0-011 workload DB identity/budget verified"
fi
# Health Check
HEALTH_PASS=0
for i in 1 2 3; do
set +e
HTTP_CODE=$(curl -sS -w "%{http_code}" -o /dev/null --connect-timeout 10 --max-time 20 "${{ env.API_HEALTH_URL }}" 2>/dev/null)
CURL_STATUS=$?
set -e
if [ "$CURL_STATUS" -ne 0 ]; then
HTTP_CODE="curl_error_${CURL_STATUS}"
fi
if [ "$HTTP_CODE" = "200" ]; then
echo "✅ API 健康檢查通過"
HEALTH_PASS=1
break
fi
echo "⏳ 嘗試 #$i: HTTP $HTTP_CODE，等待 10s..."
sleep 10
done
if [ "$HEALTH_PASS" = "0" ]; then
record_rollout_risk "public_health_final_failed"
echo "❌ API 健康檢查失敗"
exit 1
fi
SOURCE_REVISION="$SOURCE_REVISION" python3 - <<'PY'
import json
import os
import sys
import time
import urllib.request
expected = os.environ["SOURCE_REVISION"].strip().lower()
base_url = "https://awoooi.wooo.work/api/v1/agents/agent-autonomous-runtime-control"
attempts = int(
os.environ.get("EXECUTOR_BOUNDARY_READBACK_ATTEMPTS", "60")
)
last_status = "unavailable"
for attempt in range(1, attempts + 1):
url = (
f"{base_url}?expected_source_sha={expected}"
f"&readback_attempt={attempt}"
)
try:
with urllib.request.urlopen(url, timeout=10) as response:
payload = json.load(response)
except Exception as exc:
last_status = f"fetch_failed:{type(exc).__name__}"
else:
boundary = payload.get("executor_trust_boundary") or {}
database_boundary = (
boundary.get("database_identity_boundary") or {}
)
single_writer_source_boundary = (
boundary.get(
"autonomous_single_writer_source_boundary"
)
or {}
)
independent_post_verifier_source_boundary = (
boundary.get(
"independent_post_verifier_source_boundary"
)
or {}
)
canonical_learning_source_boundary = (
boundary.get("canonical_learning_source_boundary")
or {}
)
last_status = (
f"status={boundary.get('status') or 'missing'};"
"verified_source_sha="
f"{str(boundary.get('verified_source_sha') or '')[:10]};"
"deployed_source_sha="
f"{str(boundary.get('deployed_source_sha') or '')[:10]}"
)
if (
boundary.get("production_boundary_verified") is True
and boundary.get("full_workload_boundary_verified") is True
and database_boundary.get("db_identity_isolated") is True
and database_boundary.get("connection_budget_verified") is True
and single_writer_source_boundary.get(
"source_boundary_verified"
) is True
and independent_post_verifier_source_boundary.get(
"source_boundary_verified"
) is True
and canonical_learning_source_boundary.get(
"source_boundary_verified"
) is True
and str(boundary.get("verified_source_sha") or "").lower()
== expected
):
print("executor_boundary_public_readback=verified_ready")
break
if attempt == 1 or attempt % 6 == 0:
print(
"executor_boundary_public_readback_waiting="
f"{attempt}/{attempts};{last_status}",
file=sys.stderr,
)
if attempt < attempts:
time.sleep(5)
else:
raise SystemExit(
"executor boundary public readback failed: " + last_status
)
PY
echo "AWOOOI_SECURITY_BOUNDARY_VERIFIED=1"
DEPLOY_TERMINAL_RECORDED=1
ARGOCD_WAIT
ROLLOUT_EXIT=${PIPESTATUS[0]}
set -e
WORKLOAD_DB_VERIFY_EXIT=0
cat scripts/ops/awoooi-workload-db-identity-bootstrap.sh | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"AWOOOI_KUBECTL_SUDO=true AWOOOI_KUBECONFIG=/etc/rancher/k3s/k3s.yaml AWOOOI_KUBERNETES_SERVER='${{ env.K8S_API_SERVER }}' AWOOOI_RUN_ID='${GITEA_RUN_NUMBER:-cd}' bash -s -- verify" \
|| WORKLOAD_DB_VERIFY_EXIT=$?
if [ "$WORKLOAD_DB_VERIFY_EXIT" -ne 0 ]; then
restore_deploy_controls || true
echo "❌ workload database post-deploy verifier failed"
exit "$WORKLOAD_DB_VERIFY_EXIT"
fi
SECURITY_BOUNDARY_VERIFIED="0"
DEPLOY_TERMINAL_RECEIPT=$(grep '^AWOOOI_DEPLOY_TERMINAL_RECEIPT=' \
"$ROLLOUT_LOG" | tail -1 | \
sed 's/^AWOOOI_DEPLOY_TERMINAL_RECEIPT=//' || true)
if grep -q '^AWOOOI_SECURITY_BOUNDARY_VERIFIED=1$' "$ROLLOUT_LOG"; then
SECURITY_BOUNDARY_VERIFIED="1"
fi
if [ "$SECURITY_BOUNDARY_VERIFIED" != "1" ]; then
restore_deploy_controls || true
if [ -n "$DEPLOY_TERMINAL_RECEIPT" ]; then
echo "❌ deploy verifier terminal receipt: ${DEPLOY_TERMINAL_RECEIPT}"
else
echo "❌ executor security boundary did not reach its terminal verifier"
fi
if [ "$ROLLOUT_EXIT" -eq 0 ]; then
exit 1
fi
exit "$ROLLOUT_EXIT"
fi
NETWORK_POLICY_CHANGE_ACTIVE="0"
ROLLOUT_RISK="0"
ROLLOUT_SUMMARY=""
if grep -q '^AWOOOI_ROLLOUT_RISK=1$' "$ROLLOUT_LOG"; then
ROLLOUT_RISK="1"
ROLLOUT_SUMMARY=$(grep '^AWOOOI_ROLLOUT_SUMMARY=' "$ROLLOUT_LOG" | tail -1 | sed 's/^AWOOOI_ROLLOUT_SUMMARY=//' | cut -c1-700)
fi
if [ -n "${GITHUB_ENV:-}" ]; then
{
echo "AWOOI_ROLLOUT_RISK=${ROLLOUT_RISK}"
echo "AWOOI_ROLLOUT_SUMMARY=${ROLLOUT_SUMMARY}"
} >> "$GITHUB_ENV"
fi
rm -f "$ROLLOUT_LOG"
DEPLOY_READBACK_EXIT=0
python3 - <<'PY' || DEPLOY_READBACK_EXIT=$?
import json
import os
import sys
import time
import urllib.request
expected = os.environ["IMAGE_TAG"].strip().lower()
expected_short = expected[:10]
url = "https://awoooi.wooo.work/api/v1/agents/delivery-closure-workbench"
attempts = int(os.environ.get("DEPLOY_READBACK_ATTEMPTS", "36"))
sleep_seconds = int(os.environ.get("DEPLOY_READBACK_SLEEP_SECONDS", "10"))
last_error = ""
for attempt in range(1, attempts + 1):
try:
with urllib.request.urlopen(url, timeout=20) as response:
payload = json.load(response)
except Exception as exc:
last_error = f"fetch_failed={type(exc).__name__}"
print(
"production_deploy_readback_attempt="
f"{attempt}/{attempts};{last_error}",
file=sys.stderr,
)
else:
summary = payload.get("summary") if isinstance(payload, dict) else {}
if not isinstance(summary, dict):
summary = {}
runtime_short = str(
summary.get("production_deploy_runtime_build_commit_short_sha")
or ""
)
desired_short = str(
summary.get(
"production_deploy_desired_main_api_image_tag_short_sha"
)
or ""
)
desired_status = str(
summary.get(
"production_deploy_desired_main_api_image_tag_readback_status"
)
or ""
)
matches_main = (
summary.get("production_deploy_image_tag_matches_main") is True
)
if (
runtime_short == expected_short
and desired_short == expected_short
and desired_status == "ok"
):
print(
"✅ Production deploy readback matches this build and "
"GitOps desired image tag "
f"({expected_short}) on attempt {attempt}/{attempts};"
f"matches_main={matches_main}"
)
raise SystemExit(0)
last_error = (
f"expected={expected_short};runtime={runtime_short};"
f"desired={desired_short};desired_status={desired_status};"
f"matches_main={matches_main}"
)
print(
"production_deploy_readback_attempt="
f"{attempt}/{attempts};{last_error}",
file=sys.stderr,
)
if attempt < attempts:
time.sleep(sleep_seconds)
print(
"production_deploy_readback_mismatch=" + last_error,
file=sys.stderr,
)
raise SystemExit(1)
PY
if [ "$DEPLOY_READBACK_EXIT" -ne 0 ]; then
restore_deploy_controls || true
exit "$DEPLOY_READBACK_EXIT"
fi
if [ "$ROLLOUT_EXIT" -ne 0 ]; then
if [ "$ROLLOUT_RISK" = "1" ]; then
ROLLOUT_SUMMARY="${ROLLOUT_SUMMARY}; rollout_exit=${ROLLOUT_EXIT}; production_deploy_readback_matched=true"
else
ROLLOUT_RISK="1"
ROLLOUT_SUMMARY="rollout_exit=${ROLLOUT_EXIT}; production_deploy_readback_matched=true"
fi
ROLLOUT_SUMMARY=$(printf '%s' "$ROLLOUT_SUMMARY" | cut -c1-700)
if [ -n "${GITHUB_ENV:-}" ]; then
{
echo "AWOOI_ROLLOUT_RISK=${ROLLOUT_RISK}"
echo "AWOOI_ROLLOUT_SUMMARY=${ROLLOUT_SUMMARY}"
} >> "$GITHUB_ENV"
fi
echo "⚠️ ArgoCD/rollout wait exited ${ROLLOUT_EXIT}, but production deploy readback matched; treating as rollout risk, not deploy failure."
fi
if [ "$ROLLOUT_RISK" = "1" ]; then
ACTOR="${GITHUB_ACTOR:-${{ github.actor }}}"
if AWOOI_CICD_STATUS=pending \
AWOOI_CICD_STAGE=rollout-risk \
AWOOI_CICD_JOB_NAME="AWOOOI 部署完成但仍有風險證據" \
AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
AWOOI_CICD_SUMMARY="${ROLLOUT_SUMMARY}" \
scripts/ci/notify-awoooi-cicd.sh; then
echo "✅ CI/CD rollout risk notification mirrored through AWOOI API"
else
echo "⚠️ CI/CD rollout risk notification failed (non-fatal)"
fi
fi
exit 0
- name: Apply Runtime Image Mirror Policy
run: |
set -euo pipefail
prepare_deploy_key() {
mkdir -p "${HOME}/.ssh"
umask 077
local source_key="${AWOOOI_DEPLOY_SSH_KEY_PATH:-${HOME}/.ssh/deploy_key}"
if [ ! -r "${source_key}" ]; then
echo "BLOCKER runtime_image_policy_deploy_key_missing"
exit 1
fi
if [ "${source_key}" != "${HOME}/.ssh/deploy_key" ]; then
cp "${source_key}" "${HOME}/.ssh/deploy_key"
fi
chmod 600 "${HOME}/.ssh/deploy_key"
}
prepare_deploy_key
DEPLOY_KNOWN_HOSTS="${HOME}/.ssh/deploy_known_hosts"
ssh-keyscan -T 5 -t ed25519,rsa,ecdsa "${K8S_SSH_HOST}" \
> "${DEPLOY_KNOWN_HOSTS}" 2>/dev/null
test -s "${DEPLOY_KNOWN_HOSTS}" || {
echo "BLOCKER runtime_image_policy_host_key_unavailable"
exit 1
}
RECEIPT_PATH="/tmp/awoooi-runtime-image-mirror-receipt.json"
test -s "${RECEIPT_PATH}" || {
echo "BLOCKER runtime_image_mirror_receipt_missing"
exit 1
}
TRACE_ID="aia-p0-006-02a-${GITHUB_RUN_ID:-${GITEA_RUN_NUMBER:-unknown}}"
BUNDLE_DIR="$(mktemp -d)"
trap 'rm -rf "${BUNDLE_DIR}"' EXIT
cp scripts/security/runtime_image_mirror_controller.py \
"${BUNDLE_DIR}/controller.py"
cp config/security/runtime-image-mirror-policy.json \
"${BUNDLE_DIR}/policy.json"
cp "${RECEIPT_PATH}" "${BUNDLE_DIR}/mirror-receipt.json"
tar -C "${BUNDLE_DIR}" -cf - . | \
ssh \
-i "${HOME}/.ssh/deploy_key" \
-o BatchMode=yes \
-o StrictHostKeyChecking=yes \
-o UserKnownHostsFile="${DEPLOY_KNOWN_HOSTS}" \
-o ConnectTimeout=10 \
"wooo@${K8S_SSH_HOST}" \
"set -euo pipefail; TMP_DIR=\$(mktemp -d); trap 'rm -rf \"\${TMP_DIR}\"' EXIT; tar -xf - -C \"\${TMP_DIR}\"; python3 \"\${TMP_DIR}/controller.py\" --policy \"\${TMP_DIR}/policy.json\" --trace-id '${TRACE_ID}' apply --apply --mirror-receipt \"\${TMP_DIR}/mirror-receipt.json\" --kubectl-sudo --kubeconfig /etc/rancher/k3s/k3s.yaml --server '${K8S_API_SERVER}'"
echo "runtime_image_policy_stage=verified"
- name: Notify Build Deploy Success
run: |
END_TIME=$(date +%s)
DURATION=$((END_TIME - ${{ steps.commit.outputs.start_time }}))
ACTOR="${{ github.actor }}"
if AWOOI_CICD_STATUS=success \
AWOOI_CICD_STAGE=build-and-deploy \
AWOOI_CICD_JOB_NAME="AWOOOI 建置部署完成" \
AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
AWOOI_CICD_DURATION_SECONDS="${DURATION}" \
AWOOI_CICD_SUMMARY="Image build/push + ArgoCD rollout + API health passed" \
scripts/ci/notify-awoooi-cicd.sh; then
echo "✅ CI/CD build-deploy success notification mirrored through AWOOI API"
else
echo "⚠️ CI/CD build-deploy success notification failed (non-fatal)"
fi
# 2026-04-09 Claude Sonnet 4.6: Sprint 5.2 — 同步 ops 腳本到 188 (ollama user)
# 188 deploy key is rotated and must not be read by this disabled step.
# 腳本: docker-health-monitor.sh + pg-backup.sh + notify-awoooi-ops.sh
# 感知層與備份通知都先走 AWOOI API/AwoooP，Telegram 直發只保留 API 離線 fallback。
- name: Sync Ops Scripts to 188
# 2026-05-13 Codex T14e/P0:
# Disabled until the 188 ops sync path is moved to a file-secret or
# Ansible-controlled channel. Gitea Actions logs step env values, and
# multiline SSH secrets must not be exposed through CD logs.
if: ${{ false }}
continue-on-error: true
run: |
echo "188 ops script sync disabled pending secure key rotation path"
- name: Notify Pipeline Failure
if: failure()
env:
TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
run: |
COMMIT_MSG="${{ steps.commit.outputs.message }}"
SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
ACTOR="${{ github.actor }}"
FAILURE_SUMMARY="${AWOOI_ROLLOUT_SUMMARY:-${COMMIT_MSG}}"
COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&/g; s//\>/g')
MSG=$(printf '❌ AWOOOI 部署失敗\n├ 📝 %s\n├ 🔖 %s\n├ 👤 %s\n├ 🏗️ Stage: build-and-deploy\n└ 🔗 http://192.168.0.110:3001/wooo/awoooi/actions' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}")
if AWOOI_CICD_STATUS=failed \
AWOOI_CICD_STAGE=build-and-deploy \
AWOOI_CICD_JOB_NAME="AWOOOI 部署失敗" \
AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
AWOOI_CICD_SUMMARY="${FAILURE_SUMMARY}" \
scripts/ci/notify-awoooi-cicd.sh; then
echo "✅ CI/CD build failure notification mirrored through AWOOI API"
else
echo "AWOOI API notify failed; direct Telegram fallback disabled to preserve AwoooP receipt chain"
fi
post-deploy-checks:
# 2026-06-28 Codex: post-deploy checks belong to real deploy runs; skip
# marker/no-op commits already accounted for by the previous deploy run.
# 2026-07-01 Codex: `[metadata-only]` commits do not roll a new image, so
# post-deploy smokes would only retest the previous production artifact.
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]')) }}
needs: [build-and-deploy]
timeout-minutes: 30
# 2026-04-30 Codex: keep post-deploy on the host runner too. Playwright
# install-deps can also kill the act-managed job container with RWLayer=nil.
runs-on: awoooi-non110-host
steps:
- name: Bootstrap Host Runner Tools
# 2026-05-05 Codex: post-deploy also uses checkout and curl-based
# notifications, so it needs the same runner bootstrap as earlier jobs.
run: |
if command -v apk >/dev/null 2>&1; then
apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
fi
- name: Checkout source from Gitea
env:
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
run: |
set -euo pipefail
git init .
git remote remove origin 2>/dev/null || true
git remote add origin "$GITEA_SOURCE_URL"
git fetch --no-tags --prune --depth=1 origin "$GITHUB_SHA"
git checkout --force --detach FETCH_HEAD
- name: Wait for Host Web Build Pressure
# 2026-06-28 Codex: post-deploy is browser-heavy; fail closed on host
# pressure until runner load is isolated from production.
run: bash scripts/ci/wait-host-web-build-pressure.sh
- name: Get Commit Info
id: commit
run: |
echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
echo "message=$(git log -1 --pretty=%s | head -c 50)" >> $GITHUB_OUTPUT
echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT
- name: Notify Post Deploy Checks Start
run: |
ACTOR="${{ github.actor }}"
if AWOOI_CICD_STATUS=running \
AWOOI_CICD_STAGE=post-deploy-checks \
AWOOI_CICD_JOB_NAME="AWOOOI 部署後驗證開始" \
AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
AWOOI_CICD_SUMMARY="Alert Chain / Source Link / Monitoring / Smoke gates started" \
scripts/ci/notify-awoooi-cicd.sh; then
echo "✅ CI/CD post-deploy start notification mirrored through AWOOI API"
else
echo "⚠️ CI/CD post-deploy start notification failed (non-fatal)"
fi
# Phase O-4.5 2026-04-02: Alert Chain Smoke Test (Wave A.6 + B.2 ADR-037)
# 驗證告警鏈路 E2E: API Health + Webhook + OTEL + Event Exporter
# 2026-04-05 Claude Code cache優化: 使用 /opt/api-venv (已有 requests)，移除 Setup Python Tools step
# 2026-06-28 Codex: commander controlled automation keeps the canary
# evidence and notification signal, but no longer blocks CD completion.
- name: Alert Chain Smoke Test
id: alert_chain_smoke
run: |
prepare_deploy_key() {
mkdir -p "${HOME}/.ssh"
umask 077
local source_key="${AWOOOI_DEPLOY_SSH_KEY_PATH:-${HOME}/.ssh/deploy_key}"
if [ ! -r "${source_key}" ]; then
echo "❌ deploy ssh key file missing: ${source_key}" >&2
exit 1
fi
if [ "${source_key}" != "${HOME}/.ssh/deploy_key" ]; then
cp "${source_key}" "${HOME}/.ssh/deploy_key"
fi
chmod 600 "${HOME}/.ssh/deploy_key"
}
collect_observability_statuses() {
local component="$1"
ssh $SSH_OPTS "wooo@${K8S_SSH_HOST}" \
"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${K8S_API_SERVER} get pods -n observability -l app.kubernetes.io/name=${component} --no-headers -o custom-columns=STATUS:.status.phase"
}
capture_observability_statuses() {
local component="$1"
local output
if output="$(collect_observability_statuses "${component}" 2>&1)"; then
printf '%s' "${output}"
return 0
fi
printf '%s' "${output}"
return 1
}
# 2026-05-19 Codex: the smoke test runs inside CI_IMAGE, but the
# observability pod checks need the K3s host kubectl context. Capture
# those read-only statuses on the host and pass them into the
# container, instead of making the container own kube credentials.
OBSERVABILITY_PREFLIGHT_ERROR=""
OTEL_COLLECTOR_ERROR=""
EVENT_EXPORTER_ERROR=""
OTEL_COLLECTOR_STATUSES=""
EVENT_EXPORTER_STATUSES=""
prepare_deploy_key
DEPLOY_KNOWN_HOSTS="${HOME}/.ssh/deploy_known_hosts"
if ssh-keyscan -T 5 -t ed25519,rsa,ecdsa "${K8S_SSH_HOST}" > "${DEPLOY_KNOWN_HOSTS}" 2>/dev/null && test -s "${DEPLOY_KNOWN_HOSTS}"; then
SSH_OPTS="-i ${HOME}/.ssh/deploy_key -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${DEPLOY_KNOWN_HOSTS} -o ConnectTimeout=10"
if ! OTEL_COLLECTOR_STATUSES="$(capture_observability_statuses otel-collector)"; then
OTEL_COLLECTOR_ERROR="$(printf '%s' "${OTEL_COLLECTOR_STATUSES}" | tail -1 | head -c 200)"
OTEL_COLLECTOR_STATUSES=""
fi
if ! EVENT_EXPORTER_STATUSES="$(capture_observability_statuses event-exporter)"; then
EVENT_EXPORTER_ERROR="$(printf '%s' "${EVENT_EXPORTER_STATUSES}" | tail -1 | head -c 200)"
EVENT_EXPORTER_STATUSES=""
fi
else
OBSERVABILITY_PREFLIGHT_ERROR="K8s host keyscan failed"
OTEL_COLLECTOR_ERROR="${OBSERVABILITY_PREFLIGHT_ERROR}"
EVENT_EXPORTER_ERROR="${OBSERVABILITY_PREFLIGHT_ERROR}"
fi
SOURCE_LINK_RUN_REF="gitea-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}"
SOURCE_LINK_CANARY_WORK_ITEM_ID="source-evidence:sentry:upstream_canary:awoooi-source-link-canary-${SOURCE_LINK_RUN_REF}"
SOURCE_LINK_CANARY_EVENT_ID="sentry:source_correlation_linked:awoooi-source-link-canary-${SOURCE_LINK_RUN_REF}"
echo "source_link_canary_work_item_id=${SOURCE_LINK_CANARY_WORK_ITEM_ID}" >> "$GITHUB_OUTPUT"
echo "source_link_canary_event_id=${SOURCE_LINK_CANARY_EVENT_ID}" >> "$GITHUB_OUTPUT"
AWOOOP_OPERATOR_API_KEY="$(
ssh $SSH_OPTS "wooo@${K8S_SSH_HOST}" \
"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${K8S_API_SERVER} get secret awoooi-secrets -n awoooi-prod -o jsonpath='{.data.AWOOOP_OPERATOR_API_KEY}' | base64 -d"
)"
if [ -z "${AWOOOP_OPERATOR_API_KEY}" ]; then
echo "❌ AWOOOP_OPERATOR_API_KEY missing from production secret; source-link canary cannot run"
echo "alert_chain_status=fail" >> $GITHUB_OUTPUT
exit 0
fi
export AWOOOP_OPERATOR_API_KEY
# 2026-05-05 Codex: use the keepalived VIP instead of a fixed node.
# Host runner launches the CI image explicitly to avoid act RWLayer=nil.
if docker run --rm \
--name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-alert-smoke" \
--cpus "1.0" \
--memory "1g" \
-v "$PWD:/workspace" \
-v awoooi-api-venv-cache:/opt/api-venv \
-w /workspace \
-e AWOOOI_OTEL_COLLECTOR_STATUSES="${OTEL_COLLECTOR_STATUSES}" \
-e AWOOOI_OTEL_COLLECTOR_ERROR="${OTEL_COLLECTOR_ERROR}" \
-e AWOOOI_EVENT_EXPORTER_STATUSES="${EVENT_EXPORTER_STATUSES}" \
-e AWOOOI_EVENT_EXPORTER_ERROR="${EVENT_EXPORTER_ERROR}" \
-e AWOOOP_OPERATOR_API_KEY \
-e AWOOOP_OPERATOR_ID="gitea-cd-post-deploy" \
-e SOURCE_LINK_RUN_REF="${SOURCE_LINK_RUN_REF}" \
"${{ env.CI_IMAGE }}" \
bash -lc 'set -o pipefail; source /opt/api-venv/bin/activate && python3 scripts/alert_chain_smoke_test.py --api-url ${{ env.ALERT_CHAIN_API_URL }} --source-link-canary-target-incident-id INC-20260505-25E744 --run-ref "${SOURCE_LINK_RUN_REF}" --json | tee /tmp/alert_chain_result.json'; then
echo "alert_chain_status=pass" >> $GITHUB_OUTPUT
else
echo "alert_chain_status=fail" >> $GITHUB_OUTPUT
echo "⚠️ Alert Chain smoke failed; continuing under commander controlled automation"
exit 0
fi
# Phase O-5 Wave C.2 2026-04-02 ogt: 監控覆蓋率驗證 (generate_monitoring.py --check)
# 2026-06-28 Codex: coverage remains measured and notified, but no longer
# turns a deployed runtime into a blocked terminal CD state by default.
- name: Monitoring Coverage Check
id: monitoring_coverage
run: |
if docker run --rm \
--name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-coverage" \
--cpus "1.0" \
--memory "1g" \
-v "$PWD:/workspace" \
-v awoooi-api-venv-cache:/opt/api-venv \
-w /workspace \
"${{ env.CI_IMAGE }}" \
bash -lc 'source /opt/api-venv/bin/activate && python3 scripts/generate_monitoring.py --check'; then
echo "coverage_status=pass" >> $GITHUB_OUTPUT
else
echo "coverage_status=fail" >> $GITHUB_OUTPUT
echo "⚠️ Monitoring coverage check failed; continuing under commander controlled automation"
exit 0
fi
- name: AwoooP Source Correlation Applied-Link Smoke
id: source_correlation_apply_smoke
run: |
SOURCE_LINK_CANARY_WORK_ITEM_ID="${{ steps.alert_chain_smoke.outputs.source_link_canary_work_item_id }}"
SOURCE_LINK_CANARY_EVENT_ID="${{ steps.alert_chain_smoke.outputs.source_link_canary_event_id }}"
export SOURCE_LINK_CANARY_WORK_ITEM_ID SOURCE_LINK_CANARY_EVENT_ID
SOURCE_LINK_SMOKE_TIMEOUT_SECONDS="${SOURCE_LINK_SMOKE_TIMEOUT_SECONDS:-180}"
SOURCE_LINK_SMOKE_STATUS=0
if command -v timeout >/dev/null 2>&1; then
timeout -k 20s "${SOURCE_LINK_SMOKE_TIMEOUT_SECONDS}s" docker run --rm \
--name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-source-link-smoke" \
--cpus "0.5" \
--memory "512m" \
-v "$PWD:/workspace" \
-v awoooi-api-venv-cache:/opt/api-venv \
-w /workspace \
-e SOURCE_LINK_CANARY_WORK_ITEM_ID \
-e SOURCE_LINK_CANARY_EVENT_ID \
"${{ env.CI_IMAGE }}" \
bash -lc 'set -o pipefail; source /opt/api-venv/bin/activate && python3 scripts/awooop_source_correlation_apply_smoke.py \
--api-url ${{ env.ALERT_CHAIN_API_URL }} \
--target-incident-id INC-20260505-25E744 \
--work-item-id "${SOURCE_LINK_CANARY_WORK_ITEM_ID}" \
--expected-source-event-provider-event-id "${SOURCE_LINK_CANARY_EVENT_ID}" \
--allow-existing-apply \
--refresh-if-stale-days 6 \
--refresh-work-item-id "${SOURCE_LINK_CANARY_WORK_ITEM_ID}" \
--verify-refresh-candidate \
--reviewer-id gitea_cd_source_link_canary \
--operator-note "CD dedicated source-link canary; append-only status-chain proof" \
| tee /tmp/source_correlation_apply_smoke.json' || SOURCE_LINK_SMOKE_STATUS=$?
else
docker run --rm \
--name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-source-link-smoke" \
--cpus "0.5" \
--memory "512m" \
-v "$PWD:/workspace" \
-v awoooi-api-venv-cache:/opt/api-venv \
-w /workspace \
-e SOURCE_LINK_CANARY_WORK_ITEM_ID \
-e SOURCE_LINK_CANARY_EVENT_ID \
"${{ env.CI_IMAGE }}" \
bash -lc 'set -o pipefail; source /opt/api-venv/bin/activate && python3 scripts/awooop_source_correlation_apply_smoke.py \
--api-url ${{ env.ALERT_CHAIN_API_URL }} \
--target-incident-id INC-20260505-25E744 \
--work-item-id "${SOURCE_LINK_CANARY_WORK_ITEM_ID}" \
--expected-source-event-provider-event-id "${SOURCE_LINK_CANARY_EVENT_ID}" \
--allow-existing-apply \
--refresh-if-stale-days 6 \
--refresh-work-item-id "${SOURCE_LINK_CANARY_WORK_ITEM_ID}" \
--verify-refresh-candidate \
--reviewer-id gitea_cd_source_link_canary \
--operator-note "CD dedicated source-link canary; append-only status-chain proof" \
| tee /tmp/source_correlation_apply_smoke.json' || SOURCE_LINK_SMOKE_STATUS=$?
fi
echo "source_correlation_apply_rc=${SOURCE_LINK_SMOKE_STATUS}" >> "$GITHUB_OUTPUT"
if [ "${SOURCE_LINK_SMOKE_STATUS}" = "0" ]; then
echo "source_correlation_apply_status=pass" >> $GITHUB_OUTPUT
else
echo "source_correlation_apply_status=fail" >> $GITHUB_OUTPUT
echo "⚠️ Source correlation applied-link smoke failed; continuing under commander controlled automation rc=${SOURCE_LINK_SMOKE_STATUS}"
exit 0
fi
# [首席架構師] 新增 Playwright E2E Smoke Test 步驟 v1.0.0 2026-04-01 (台北時間)
# continue-on-error: true — smoke 失敗不阻塞部署，但結果會反映在 TG 通知
- name: E2E Smoke Test
id: smoke
continue-on-error: true
run: |
cat > /tmp/awoooi-smoke.sh <<'CI_SCRIPT'
set -euo pipefail
# 首席架構師 Review I4 + 2026-04-05 Claude Code cache優化:
# playwright.config.ts import @playwright/test — 必須先安裝 pnpm node_modules
# pnpm store 持久化到 /opt/pnpm-store，pnpm-lock.yaml hash 未變則 --prefer-offline
SOURCE_WORKDIR=/source
SMOKE_WORKDIR=/tmp/awoooi-smoke-workspace
cleanup_smoke_workspace_artifacts() {
rm -rf "$SMOKE_WORKDIR" \
/tmp/pnpm-install.log \
/tmp/playwright-install-deps.log \
2>/dev/null || true
}
trap cleanup_smoke_workspace_artifacts EXIT
rm -rf "$SMOKE_WORKDIR"
mkdir -p "$SMOKE_WORKDIR"
if command -v tar >/dev/null 2>&1; then
tar \
--exclude='./.git' \
--exclude='./node_modules' \
--exclude='./apps/web/node_modules' \
--exclude='./apps/web/test-results' \
--exclude='./apps/web/playwright-report' \
--exclude='./packages/*/node_modules' \
-cf - -C "$SOURCE_WORKDIR" . | tar -xf - -C "$SMOKE_WORKDIR"
else
cp -a "$SOURCE_WORKDIR/." "$SMOKE_WORKDIR/"
rm -rf "$SMOKE_WORKDIR/.git" \
"$SMOKE_WORKDIR/node_modules" \
"$SMOKE_WORKDIR/apps/web/node_modules" \
"$SMOKE_WORKDIR/apps/web/test-results" \
"$SMOKE_WORKDIR/apps/web/playwright-report" \
2>/dev/null || true
fi
cd "$SMOKE_WORKDIR"
PNPM_STORE=/opt/pnpm-store
PNPM_HASH_FILE=/opt/pnpm-store/.lock_hash
CURRENT_PNPM_HASH=$(md5sum pnpm-lock.yaml | awk '{print $1}')
corepack enable 2>/dev/null || npm install -g pnpm@9 -q
mkdir -p "$PNPM_STORE"
pnpm config set store-dir $PNPM_STORE
if [ "$(cat $PNPM_HASH_FILE 2>/dev/null)" != "$CURRENT_PNPM_HASH" ]; then
echo "📦 pnpm lock 已變更，重裝 node_modules..."
pnpm install --frozen-lockfile 2>&1 | tee /tmp/pnpm-install.log | tail -20
echo "$CURRENT_PNPM_HASH" > $PNPM_HASH_FILE
else
echo "⚡ 使用快取 pnpm store (lock 未變更)，prefer-offline..."
pnpm install --frozen-lockfile --prefer-offline 2>&1 | tee /tmp/pnpm-install.log | tail -20
fi
pnpm --dir apps/web exec node -e "require.resolve('@playwright/test')"
cd apps/web
# Playwright Chromium 持久化到 /opt/playwright-browsers，版本 hash guard
export PLAYWRIGHT_BROWSERS_PATH=/opt/playwright-browsers
PLAYWRIGHT_VER=$(node -e "console.log(require('./package.json').devDependencies['@playwright/test'] || '')" 2>/dev/null || echo "unknown")
PLAYWRIGHT_HASH_FILE=/opt/playwright-browsers/.version_hash
BROWSER_PATH=$(node -e "const { chromium } = require('@playwright/test'); console.log(chromium.executablePath())")
if [ "$(cat $PLAYWRIGHT_HASH_FILE 2>/dev/null)" != "$PLAYWRIGHT_VER" ] || [ ! -x "$BROWSER_PATH" ]; then
echo "📦 Playwright browser cache missing/stale ($PLAYWRIGHT_VER): $BROWSER_PATH"
pnpm exec playwright install chromium --with-deps 2>&1 | tail -20
BROWSER_PATH=$(node -e "const { chromium } = require('@playwright/test'); console.log(chromium.executablePath())")
test -x "$BROWSER_PATH" || { echo "❌ Playwright browser executable missing after install: $BROWSER_PATH"; exit 1; }
echo "$PLAYWRIGHT_VER" > $PLAYWRIGHT_HASH_FILE
else
echo "⚡ 使用快取 Playwright Chromium ($PLAYWRIGHT_VER): $BROWSER_PATH"
fi
# Browser cache 命中時也要確認 OS shared libs 存在；否則 smoke 會只測到
# chromium launch failure（例如 libnspr4.so missing）。
if ! ldconfig -p 2>/dev/null | grep -q 'libnspr4'; then
echo "📦 Playwright system deps missing，補安裝 Chromium deps..."
pnpm exec playwright install-deps chromium > /tmp/playwright-install-deps.log 2>&1 || {
tail -40 /tmp/playwright-install-deps.log
exit 1
}
tail -20 /tmp/playwright-install-deps.log
fi
# 對已部署的生產環境跑 smoke test
SMOKE_STATUS=pass
pnpm exec playwright test tests/e2e/smoke.spec.ts --reporter=line || SMOKE_STATUS=fail
echo "smoke_status=${SMOKE_STATUS}" >> $GITHUB_OUTPUT
CI_SCRIPT
SMOKE_OUTPUT="$PWD/.awoooi-smoke-output"
rm -f "$SMOKE_OUTPUT"
touch "$SMOKE_OUTPUT"
chmod 666 "$SMOKE_OUTPUT"
SMOKE_DOCKER_STATUS=0
# 2026-06-01 Codex: post-deploy smoke can pass, then hang in
# runner cleanup and incorrectly mark the deploy failed. Bound only
# the smoke container; preserve pass evidence if it was written.
if command -v timeout >/dev/null 2>&1; then
# 2026-06-14 Codex: act-runner host may provide BusyBox timeout,
# which rejects GNU-only --kill-after. The short -k form works
# with BusyBox and GNU timeout.
timeout -k 20s 300s docker run --rm \
--name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-e2e-smoke" \
--cpus "1.5" \
--memory "2g" \
-v "$PWD:/source:ro" \
-v "$SMOKE_OUTPUT:/github-output" \
-v /tmp/awoooi-smoke.sh:/tmp/awoooi-smoke.sh:ro \
-v awoooi-pnpm-store:/opt/pnpm-store \
-v awoooi-playwright-browsers:/opt/playwright-browsers \
-w /tmp \
-e GITHUB_OUTPUT=/github-output \
-e CI=true \
-e PLAYWRIGHT_BASE_URL=https://awoooi.wooo.work \
"${{ env.CI_IMAGE }}" \
bash /tmp/awoooi-smoke.sh || SMOKE_DOCKER_STATUS=$?
else
docker run --rm \
--name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-e2e-smoke" \
--cpus "1.5" \
--memory "2g" \
-v "$PWD:/source:ro" \
-v "$SMOKE_OUTPUT:/github-output" \
-v /tmp/awoooi-smoke.sh:/tmp/awoooi-smoke.sh:ro \
-v awoooi-pnpm-store:/opt/pnpm-store \
-v awoooi-playwright-browsers:/opt/playwright-browsers \
-w /tmp \
-e GITHUB_OUTPUT=/github-output \
-e CI=true \
-e PLAYWRIGHT_BASE_URL=https://awoooi.wooo.work \
"${{ env.CI_IMAGE }}" \
bash /tmp/awoooi-smoke.sh || SMOKE_DOCKER_STATUS=$?
fi
if [ "$SMOKE_DOCKER_STATUS" != "0" ] && ! grep -q '^smoke_status=pass$' "$SMOKE_OUTPUT"; then
echo "smoke_status=fail" > "$SMOKE_OUTPUT"
echo "E2E smoke container failed before pass evidence: ${SMOKE_DOCKER_STATUS}"
exit "$SMOKE_DOCKER_STATUS"
fi
if [ "$SMOKE_DOCKER_STATUS" != "0" ]; then
echo "E2E smoke pass evidence was written; treating container exit ${SMOKE_DOCKER_STATUS} as cleanup timeout"
fi
cat "$SMOKE_OUTPUT" >> "$GITHUB_OUTPUT"
env:
CI: "true"
# 直接測試已部署的生產環境，不啟動本地 dev server
PLAYWRIGHT_BASE_URL: "https://awoooi.wooo.work"
- name: Notify Health Check Success
env:
TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
SMOKE_RESULT: ${{ steps.smoke.outputs.smoke_status == 'pass' && '✅' || '⚠️' }}
ALERT_CHAIN_RESULT: ${{ steps.alert_chain_smoke.outputs.alert_chain_status == 'pass' && '✅' || '⚠️' }}
MONITORING_RESULT: ${{ steps.monitoring_coverage.outputs.coverage_status == 'pass' && '✅' || '⚠️' }}
SOURCE_LINK_RESULT: ${{ steps.source_correlation_apply_smoke.outputs.source_correlation_apply_status == 'pass' && '✅' || '⚠️' }}
run: |
END_TIME=$(date +%s)
DURATION=$((END_TIME - ${{ steps.commit.outputs.start_time }}))
MINUTES=$((DURATION / 60))
SECONDS=$((DURATION % 60))
# 2026-04-05 ogt: TG_MSG 必須在 shell 中組裝，才能展開 ${MINUTES}/${SECONDS} 等 shell 變數
# 2026-04-05 ogt: 移除 parse_mode=HTML，避免 commit message 含特殊字元導致 400
COMMIT_MSG="${{ steps.commit.outputs.message }}"
SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
TG_MSG="✅ AWOOOI 部署完成\n├ 📝 ${COMMIT_MSG}\n├ 🔖 ${SHORT_SHA}\n├ ⏱️ 耗時: ${MINUTES}m ${SECONDS}s\n├ 📦 API: ✅ Web: ✅\n├ 🩺 Health: ✅\n├ 🔗 Alert Chain: ${ALERT_CHAIN_RESULT}\n├ 🧷 Source Link: ${SOURCE_LINK_RESULT}\n├ 📊 Monitoring: ${MONITORING_RESULT}\n└ 🎭 Smoke: ${SMOKE_RESULT}"
if AWOOI_CICD_STATUS=success \
AWOOI_CICD_STAGE=post-deploy \
AWOOI_CICD_JOB_NAME="AWOOOI 部署完成" \
AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
AWOOI_CICD_DURATION_SECONDS="${DURATION}" \
AWOOI_CICD_SUMMARY="API=✅; Web=✅; AlertChain=${ALERT_CHAIN_RESULT}; SourceLink=${SOURCE_LINK_RESULT}; Monitoring=${MONITORING_RESULT}; Smoke=${SMOKE_RESULT}" \
scripts/ci/notify-awoooi-cicd.sh; then
echo "✅ CI/CD success notification mirrored through AWOOI API"
else
echo "AWOOI API notify failed; direct Telegram fallback disabled to preserve AwoooP receipt chain"
fi
- name: Notify Pipeline Failure
# 2026-04-16 ogt + Claude Sonnet 4.6: 改用 HTML 結構化格式
if: failure()
env:
TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
run: |
COMMIT_MSG="${{ steps.commit.outputs.message }}"
SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
ACTOR="${{ github.actor }}"
COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&/g; s//\>/g')
MSG=$(printf '❌ AWOOOI 部署失敗\n├ 📝 %s\n├ 🔖 %s\n├ 👤 %s\n├ 🩺 Stage: post-deploy-checks\n└ 🔗 http://192.168.0.110:3001/wooo/awoooi/actions' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}")
if AWOOI_CICD_STATUS=failed \
AWOOI_CICD_STAGE=post-deploy-checks \
AWOOI_CICD_JOB_NAME="AWOOOI 部署失敗" \
AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \
scripts/ci/notify-awoooi-cicd.sh; then
echo "✅ CI/CD post-deploy failure notification mirrored through AWOOI API"
else
echo "AWOOI API notify failed; direct Telegram fallback disabled to preserve AwoooP receipt chain"
fi
- name: Clean Post-Deploy Workspace Artifacts
if: always()
env:
HOST_RUNNER_CLEANUP_IMAGE: ${{ env.CI_IMAGE }}
run: bash scripts/ci/cleanup-host-runner-workspace.sh