diff --git a/apps/api/src/main.py b/apps/api/src/main.py index 875e29313..f9ca12fce 100644 --- a/apps/api/src/main.py +++ b/apps/api/src/main.py @@ -64,8 +64,8 @@ from src.api.v1 import incidents as incidents_v1 # Phase 6.4: Decision Proposal from src.api.v1 import iwooos as iwooos_v1 # IwoooS security governance API from src.api.v1 import knowledge as knowledge_v1 # KB Phase 1: Knowledge Base from src.api.v1 import learning as learning_v1 # Phase D-G P0: Learning API -from src.api.v1 import metrics as metrics_v1 # Phase 7: Gold Metrics (真實血脈) from src.api.v1 import mcp_control_plane as mcp_control_plane_v1 +from src.api.v1 import metrics as metrics_v1 # Phase 7: Gold Metrics (真實血脈) from src.api.v1 import monitoring as monitoring_v1 # 2026-04-03: 監控工具狀態 from src.api.v1 import notifications as notifications_v1 # 2026-04-10: 通知頻道狀態 from src.api.v1 import ( @@ -650,22 +650,24 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]: logger.warning("approval_timeout_resolver_schedule_failed", error=str(e)) # T73: 已有完成證據但仍卡在 INVESTIGATING 的舊 incident 小批次收斂。 - # 僅處理 auto-repair success / approval EXECUTION_SUCCESS / approval EXPIRED， - # 不自動關閉 manual_required 或單純 APPROVED 事件。 + # API production pool 只有 serving 預算；實際 loop 由單一 standalone + # signal worker 擁有，避免 coroutine 被 budget gate 關閉後仍假性記錄 + # scheduled。reconciler 本身只處理強證據或已不再 firing 的事件。 try: from src.jobs.incident_lifecycle_reconciler import ( INTERVAL_SECONDS as INCIDENT_LIFECYCLE_RECONCILER_INTERVAL, ) - from src.jobs.incident_lifecycle_reconciler import ( - run_incident_lifecycle_reconciler_loop, - ) - schedule_api_background_task(run_incident_lifecycle_reconciler_loop()) logger.info( - "incident_lifecycle_reconciler_scheduled", + "incident_lifecycle_reconciler_owner_delegated", interval_sec=INCIDENT_LIFECYCLE_RECONCILER_INTERVAL, + scheduled_in_api_process=False, + production_process_owner="awoooi-worker", ) except Exception as e: - logger.warning("incident_lifecycle_reconciler_schedule_failed", error=str(e)) + logger.warning( + "incident_lifecycle_reconciler_owner_resolution_failed", + error=str(e), + ) # AwoooP Ansible candidate backfill worker. # 把近期已命中 allowlisted PlayBook、但缺 durable candidate row 的事故補進 diff --git a/apps/api/src/services/mcp_version_lifecycle_service.py b/apps/api/src/services/mcp_version_lifecycle_service.py index d1ecf328a..c4ed3a2c3 100644 --- a/apps/api/src/services/mcp_version_lifecycle_service.py +++ b/apps/api/src/services/mcp_version_lifecycle_service.py @@ -42,7 +42,7 @@ _SOURCE_HOSTS = { } _VERSION_PATTERNS = ( re.compile( - rb"""(?ix)["'](?:version|latestVersion|packageVersion)["']\s*[:=]\s*["']v?""" + rb"""(?ix)["'](?:latestVersion|packageVersion|softwareVersion)["']\s*[:=]\s*["']v?""" rb"""([0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?)["']""" ), re.compile( diff --git a/apps/api/src/services/telegram_alert_ai_automation_matrix.py b/apps/api/src/services/telegram_alert_ai_automation_matrix.py index 76a90e288..9e9c9fbd6 100644 --- a/apps/api/src/services/telegram_alert_ai_automation_matrix.py +++ b/apps/api/src/services/telegram_alert_ai_automation_matrix.py @@ -703,13 +703,14 @@ def _inspect_source_contract(repo_root: Path) -> dict[str, Any]: def _read_source_contract_text(repo_root: Path, relative_path: str) -> str: - candidates = [repo_root / relative_path] + source_path = repo_root / relative_path + candidates = [source_path] api_prefix = "apps/api/" if relative_path.startswith(api_prefix): candidates.append(repo_root / relative_path.removeprefix(api_prefix)) if ( relative_path == _AWOOOP_VERIFIED_CONTEXT_SURFACE_SOURCE - and not (repo_root / "apps").exists() + and not source_path.exists() ): candidates.append(repo_root / "src/services/telegram_alert_ai_automation_matrix.py") diff --git a/apps/api/src/workers/signal_worker.py b/apps/api/src/workers/signal_worker.py index 1a16c0a39..38c029333 100644 --- a/apps/api/src/workers/signal_worker.py +++ b/apps/api/src/workers/signal_worker.py @@ -623,7 +623,35 @@ async def _main() -> None: backup_restore_legacy_backfill_task: asyncio.Task[Any] | None = None security_maintenance_tasks: list[asyncio.Task[Any]] = [] agent99_controlled_dispatch_reconciler_task: asyncio.Task[Any] | None = None + incident_lifecycle_reconciler_task: asyncio.Task[Any] | None = None mcp_version_lifecycle_task: asyncio.Task[Any] | None = None + + # Production API pods reserve their DB pool for serving requests. This + # singleton worker therefore owns the existing bounded, fail-closed + # lifecycle reconciler. It resolves at most one configured batch per pass + # and independently reads current Prometheus firing state before treating + # inactive alerts as closure candidates. + from src.jobs.incident_lifecycle_reconciler import ( + BATCH_LIMIT as INCIDENT_LIFECYCLE_RECONCILER_BATCH_LIMIT, + ) + from src.jobs.incident_lifecycle_reconciler import ( + INTERVAL_SECONDS as INCIDENT_LIFECYCLE_RECONCILER_INTERVAL, + ) + from src.jobs.incident_lifecycle_reconciler import ( + run_incident_lifecycle_reconciler_loop, + ) + + incident_lifecycle_reconciler_task = asyncio.create_task( + run_incident_lifecycle_reconciler_loop(), + name="run_incident_lifecycle_reconciler_loop", + ) + logger.info( + "signal_worker_incident_lifecycle_reconciler_started", + owner="signal_worker", + interval_seconds=INCIDENT_LIFECYCLE_RECONCILER_INTERVAL, + batch_limit=INCIDENT_LIFECYCLE_RECONCILER_BATCH_LIMIT, + direct_incident_closure_performed=False, + ) if settings.ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER: from src.jobs.awooop_ansible_candidate_backfill_job import ( run_awooop_ansible_candidate_backfill_loop, @@ -794,6 +822,12 @@ async def _main() -> None: await agent99_controlled_dispatch_reconciler_task except asyncio.CancelledError: pass + if incident_lifecycle_reconciler_task is not None: + incident_lifecycle_reconciler_task.cancel() + try: + await incident_lifecycle_reconciler_task + except asyncio.CancelledError: + pass if mcp_version_lifecycle_task is not None: mcp_version_lifecycle_task.cancel() try: diff --git a/apps/api/tests/test_mcp_version_lifecycle_service.py b/apps/api/tests/test_mcp_version_lifecycle_service.py index 17c3f77de..15fc35bd4 100644 --- a/apps/api/tests/test_mcp_version_lifecycle_service.py +++ b/apps/api/tests/test_mcp_version_lifecycle_service.py @@ -90,7 +90,7 @@ async def test_lifecycle_run_persists_baselines_and_fail_closed_terminals() -> N status="ok", source_url=url, http_status=200, - body=b'{"version":"1.2.3","content":"not persisted"}', + body=b'{"packageVersion":"1.2.3","content":"not persisted"}', ) result = await run_mcp_version_lifecycle_once( @@ -127,7 +127,7 @@ async def test_lifecycle_run_persists_baselines_and_fail_closed_terminals() -> N def test_changed_version_stops_before_replay_without_immutable_receipts() -> None: - old_body = b'{"version":"1.2.3"}' + old_body = b'{"packageVersion":"1.2.3"}' candidate = _catalog()["capabilities"][0] observation = build_candidate_observation( candidate, @@ -135,7 +135,7 @@ def test_changed_version_stops_before_replay_without_immutable_receipts() -> Non status="ok", source_url=candidate["catalog_url"], http_status=200, - body=b'{"version":"1.3.0"}', + body=b'{"packageVersion":"1.3.0"}', ), previous={ "observed_version": "1.2.3", @@ -201,13 +201,20 @@ async def test_source_failure_is_durable_partial_degraded_no_write() -> None: def test_version_parser_rejects_ambiguous_page_versions() -> None: - assert _extract_unambiguous_version(b'{"version":"1.2.3"}') == "1.2.3" + assert _extract_unambiguous_version(b'{"packageVersion":"1.2.3"}') == "1.2.3" assert ( - _extract_unambiguous_version(b'{"version":"1.2.3","latestVersion":"2.0.0"}') + _extract_unambiguous_version( + b'{"packageVersion":"1.2.3","latestVersion":"2.0.0"}' + ) is None ) +def test_version_parser_ignores_cloudflare_beacon_version() -> None: + body = b"data-cf-beacon='{\"version\":\"2024.11.0\",\"token\":\"public\"}'" + assert _extract_unambiguous_version(body) is None + + def test_stable_fingerprint_ignores_randomized_recommendation_sections() -> None: stable_head = b""" Context7 MCP diff --git a/apps/api/tests/test_runtime_bootstrap_guards.py b/apps/api/tests/test_runtime_bootstrap_guards.py index f4b36be32..d42a70e7f 100644 --- a/apps/api/tests/test_runtime_bootstrap_guards.py +++ b/apps/api/tests/test_runtime_bootstrap_guards.py @@ -539,6 +539,40 @@ def test_signal_worker_gracefully_cancels_agent99_reconciler() -> None: assert source.index(await_task) < source.index("await worker.stop()") +def test_signal_worker_is_single_incident_lifecycle_reconciler_owner() -> None: + import inspect + + from src import main as api_main + from src.workers import signal_worker + + api_source = inspect.getsource(api_main.lifespan) + worker_source = inspect.getsource(signal_worker._main) + + assert "run_incident_lifecycle_reconciler_loop" not in api_source + assert "incident_lifecycle_reconciler_owner_delegated" in api_source + assert worker_source.count("run_incident_lifecycle_reconciler_loop()") == 1 + assert worker_source.index("if settings.DATABASE_BOOTSTRAP_DDL_ENABLED") < ( + worker_source.index("run_incident_lifecycle_reconciler_loop()") + ) + assert "signal_worker_incident_lifecycle_reconciler_started" in worker_source + assert 'owner="signal_worker"' in worker_source + + +def test_signal_worker_gracefully_cancels_incident_lifecycle_reconciler() -> None: + import inspect + + from src.workers import signal_worker + + source = inspect.getsource(signal_worker._main) + cancel = "incident_lifecycle_reconciler_task.cancel()" + await_task = "await incident_lifecycle_reconciler_task" + + assert cancel in source + assert await_task in source + assert source.index(cancel) < source.index(await_task) + assert source.index(await_task) < source.index("await worker.stop()") + + def test_production_signal_worker_is_single_agent99_reconciler_owner() -> None: import inspect diff --git a/apps/api/tests/test_telegram_alert_ai_automation_matrix_api.py b/apps/api/tests/test_telegram_alert_ai_automation_matrix_api.py index d4d2a5994..4ed760981 100644 --- a/apps/api/tests/test_telegram_alert_ai_automation_matrix_api.py +++ b/apps/api/tests/test_telegram_alert_ai_automation_matrix_api.py @@ -8,6 +8,7 @@ from fastapi.testclient import TestClient from src.api.v1.agents import router from src.services.telegram_alert_ai_automation_matrix import ( _inspect_source_contract, + _read_source_contract_text, load_latest_telegram_alert_ai_automation_matrix, ) @@ -210,6 +211,28 @@ def test_telegram_matrix_source_contract_supports_container_layout(tmp_path): assert proof["source_contract_missing_markers"] == [] +def test_telegram_matrix_source_contract_uses_embedded_surface_when_web_package_only( + tmp_path, +): + (tmp_path / "apps/web").mkdir(parents=True) + (tmp_path / "apps/web/package.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "src/services").mkdir(parents=True) + embedded_source = ( + tmp_path / "src/services/telegram_alert_ai_automation_matrix.py" + ) + embedded_source.write_text( + 'data-testid="telegram-alert-post-apply-verifier-readback"\n', + encoding="utf-8", + ) + + source = _read_source_contract_text( + tmp_path, + "apps/web/src/components/awooop/autonomous-runtime-receipt-panel.tsx", + ) + + assert 'data-testid="telegram-alert-post-apply-verifier-readback"' in source + + def test_telegram_matrix_source_contract_reports_missing_web_surface_without_raising( tmp_path, ): diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index f08994ce9..437cba51b 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "ad3eaa64841255744285a3bcbe1cfcae2f6abe37" + value: "29072e4004f1fa50478e63bc4e3daaee1b4d96e4" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "ad3eaa64841255744285a3bcbe1cfcae2f6abe37" + value: "29072e4004f1fa50478e63bc4e3daaee1b4d96e4" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 3ef983de9..39c3e1d57 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "ad3eaa64841255744285a3bcbe1cfcae2f6abe37" + value: "29072e4004f1fa50478e63bc4e3daaee1b4d96e4" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 5125a83a9..4d768dd13 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "ad3eaa64841255744285a3bcbe1cfcae2f6abe37" + value: "29072e4004f1fa50478e63bc4e3daaee1b4d96e4" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index aee25ed1e..24fa6e4b4 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:8f8c0e79455d0025c8f6038b11a08dfa917f0b07104a02d5abe708e70ea65385 +- digest: sha256:070d950d521b174d1472ab285f117353a56880eb6465dbabea4a7afc5e8e54ba name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:d59e143e0fa2e972de08fde7818ebaac8e2a361998ecc37a02b60c4bf10300b0 +- digest: sha256:c093bf84158a6a77306513d80b02a95d4ef2c963f08d7e097d69253d37211f12 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web