fix(runtime): restore lifecycle and Telegram verification
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m17s
CD Pipeline / build-and-deploy (push) Successful in 13m9s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s

This commit is contained in:
ogt
2026-07-15 03:27:00 +08:00
parent 915277d149
commit 05ddd1538e
5 changed files with 105 additions and 11 deletions

View File

@@ -64,8 +64,8 @@ from src.api.v1 import incidents as incidents_v1 # Phase 6.4: Decision Proposal
from src.api.v1 import iwooos as iwooos_v1 # IwoooS security governance API
from src.api.v1 import knowledge as knowledge_v1 # KB Phase 1: Knowledge Base
from src.api.v1 import learning as learning_v1 # Phase D-G P0: Learning API
from src.api.v1 import metrics as metrics_v1 # Phase 7: Gold Metrics (真實血脈)
from src.api.v1 import mcp_control_plane as mcp_control_plane_v1
from src.api.v1 import metrics as metrics_v1 # Phase 7: Gold Metrics (真實血脈)
from src.api.v1 import monitoring as monitoring_v1 # 2026-04-03: 監控工具狀態
from src.api.v1 import notifications as notifications_v1 # 2026-04-10: 通知頻道狀態
from src.api.v1 import (
@@ -650,22 +650,24 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
logger.warning("approval_timeout_resolver_schedule_failed", error=str(e))
# T73: 已有完成證據但仍卡在 INVESTIGATING 的舊 incident 小批次收斂。
# 僅處理 auto-repair success / approval EXECUTION_SUCCESS / approval EXPIRED
# 不自動關閉 manual_required 或單純 APPROVED 事件。
# API production pool 只有 serving 預算;實際 loop 由單一 standalone
# signal worker 擁有,避免 coroutine 被 budget gate 關閉後仍假性記錄
# scheduled。reconciler 本身只處理強證據或已不再 firing 的事件。
try:
from src.jobs.incident_lifecycle_reconciler import (
INTERVAL_SECONDS as INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
)
from src.jobs.incident_lifecycle_reconciler import (
run_incident_lifecycle_reconciler_loop,
)
schedule_api_background_task(run_incident_lifecycle_reconciler_loop())
logger.info(
"incident_lifecycle_reconciler_scheduled",
"incident_lifecycle_reconciler_owner_delegated",
interval_sec=INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
scheduled_in_api_process=False,
production_process_owner="awoooi-worker",
)
except Exception as e:
logger.warning("incident_lifecycle_reconciler_schedule_failed", error=str(e))
logger.warning(
"incident_lifecycle_reconciler_owner_resolution_failed",
error=str(e),
)
# AwoooP Ansible candidate backfill worker.
# 把近期已命中 allowlisted PlayBook、但缺 durable candidate row 的事故補進

View File

@@ -703,13 +703,14 @@ def _inspect_source_contract(repo_root: Path) -> dict[str, Any]:
def _read_source_contract_text(repo_root: Path, relative_path: str) -> str:
candidates = [repo_root / relative_path]
source_path = repo_root / relative_path
candidates = [source_path]
api_prefix = "apps/api/"
if relative_path.startswith(api_prefix):
candidates.append(repo_root / relative_path.removeprefix(api_prefix))
if (
relative_path == _AWOOOP_VERIFIED_CONTEXT_SURFACE_SOURCE
and not (repo_root / "apps").exists()
and not source_path.exists()
):
candidates.append(repo_root / "src/services/telegram_alert_ai_automation_matrix.py")

View File

@@ -623,7 +623,35 @@ async def _main() -> None:
backup_restore_legacy_backfill_task: asyncio.Task[Any] | None = None
security_maintenance_tasks: list[asyncio.Task[Any]] = []
agent99_controlled_dispatch_reconciler_task: asyncio.Task[Any] | None = None
incident_lifecycle_reconciler_task: asyncio.Task[Any] | None = None
mcp_version_lifecycle_task: asyncio.Task[Any] | None = None
# Production API pods reserve their DB pool for serving requests. This
# singleton worker therefore owns the existing bounded, fail-closed
# lifecycle reconciler. It resolves at most one configured batch per pass
# and independently reads current Prometheus firing state before treating
# inactive alerts as closure candidates.
from src.jobs.incident_lifecycle_reconciler import (
BATCH_LIMIT as INCIDENT_LIFECYCLE_RECONCILER_BATCH_LIMIT,
)
from src.jobs.incident_lifecycle_reconciler import (
INTERVAL_SECONDS as INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
)
from src.jobs.incident_lifecycle_reconciler import (
run_incident_lifecycle_reconciler_loop,
)
incident_lifecycle_reconciler_task = asyncio.create_task(
run_incident_lifecycle_reconciler_loop(),
name="run_incident_lifecycle_reconciler_loop",
)
logger.info(
"signal_worker_incident_lifecycle_reconciler_started",
owner="signal_worker",
interval_seconds=INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
batch_limit=INCIDENT_LIFECYCLE_RECONCILER_BATCH_LIMIT,
direct_incident_closure_performed=False,
)
if settings.ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER:
from src.jobs.awooop_ansible_candidate_backfill_job import (
run_awooop_ansible_candidate_backfill_loop,
@@ -794,6 +822,12 @@ async def _main() -> None:
await agent99_controlled_dispatch_reconciler_task
except asyncio.CancelledError:
pass
if incident_lifecycle_reconciler_task is not None:
incident_lifecycle_reconciler_task.cancel()
try:
await incident_lifecycle_reconciler_task
except asyncio.CancelledError:
pass
if mcp_version_lifecycle_task is not None:
mcp_version_lifecycle_task.cancel()
try:

View File

@@ -539,6 +539,40 @@ def test_signal_worker_gracefully_cancels_agent99_reconciler() -> None:
assert source.index(await_task) < source.index("await worker.stop()")
def test_signal_worker_is_single_incident_lifecycle_reconciler_owner() -> None:
import inspect
from src import main as api_main
from src.workers import signal_worker
api_source = inspect.getsource(api_main.lifespan)
worker_source = inspect.getsource(signal_worker._main)
assert "run_incident_lifecycle_reconciler_loop" not in api_source
assert "incident_lifecycle_reconciler_owner_delegated" in api_source
assert worker_source.count("run_incident_lifecycle_reconciler_loop()") == 1
assert worker_source.index("if settings.DATABASE_BOOTSTRAP_DDL_ENABLED") < (
worker_source.index("run_incident_lifecycle_reconciler_loop()")
)
assert "signal_worker_incident_lifecycle_reconciler_started" in worker_source
assert 'owner="signal_worker"' in worker_source
def test_signal_worker_gracefully_cancels_incident_lifecycle_reconciler() -> None:
import inspect
from src.workers import signal_worker
source = inspect.getsource(signal_worker._main)
cancel = "incident_lifecycle_reconciler_task.cancel()"
await_task = "await incident_lifecycle_reconciler_task"
assert cancel in source
assert await_task in source
assert source.index(cancel) < source.index(await_task)
assert source.index(await_task) < source.index("await worker.stop()")
def test_production_signal_worker_is_single_agent99_reconciler_owner() -> None:
import inspect

View File

@@ -8,6 +8,7 @@ from fastapi.testclient import TestClient
from src.api.v1.agents import router
from src.services.telegram_alert_ai_automation_matrix import (
_inspect_source_contract,
_read_source_contract_text,
load_latest_telegram_alert_ai_automation_matrix,
)
@@ -210,6 +211,28 @@ def test_telegram_matrix_source_contract_supports_container_layout(tmp_path):
assert proof["source_contract_missing_markers"] == []
def test_telegram_matrix_source_contract_uses_embedded_surface_when_web_package_only(
tmp_path,
):
(tmp_path / "apps/web").mkdir(parents=True)
(tmp_path / "apps/web/package.json").write_text("{}\n", encoding="utf-8")
(tmp_path / "src/services").mkdir(parents=True)
embedded_source = (
tmp_path / "src/services/telegram_alert_ai_automation_matrix.py"
)
embedded_source.write_text(
'data-testid="telegram-alert-post-apply-verifier-readback"\n',
encoding="utf-8",
)
source = _read_source_contract_text(
tmp_path,
"apps/web/src/components/awooop/autonomous-runtime-receipt-panel.tsx",
)
assert 'data-testid="telegram-alert-post-apply-verifier-readback"' in source
def test_telegram_matrix_source_contract_reports_missing_web_surface_without_raising(
tmp_path,
):