fix(runtime): restore lifecycle and Telegram verification
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m17s
CD Pipeline / build-and-deploy (push) Successful in 13m9s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m17s
CD Pipeline / build-and-deploy (push) Successful in 13m9s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s
This commit is contained in:
@@ -64,8 +64,8 @@ from src.api.v1 import incidents as incidents_v1 # Phase 6.4: Decision Proposal
|
||||
from src.api.v1 import iwooos as iwooos_v1 # IwoooS security governance API
|
||||
from src.api.v1 import knowledge as knowledge_v1 # KB Phase 1: Knowledge Base
|
||||
from src.api.v1 import learning as learning_v1 # Phase D-G P0: Learning API
|
||||
from src.api.v1 import metrics as metrics_v1 # Phase 7: Gold Metrics (真實血脈)
|
||||
from src.api.v1 import mcp_control_plane as mcp_control_plane_v1
|
||||
from src.api.v1 import metrics as metrics_v1 # Phase 7: Gold Metrics (真實血脈)
|
||||
from src.api.v1 import monitoring as monitoring_v1 # 2026-04-03: 監控工具狀態
|
||||
from src.api.v1 import notifications as notifications_v1 # 2026-04-10: 通知頻道狀態
|
||||
from src.api.v1 import (
|
||||
@@ -650,22 +650,24 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
logger.warning("approval_timeout_resolver_schedule_failed", error=str(e))
|
||||
|
||||
# T73: 已有完成證據但仍卡在 INVESTIGATING 的舊 incident 小批次收斂。
|
||||
# 僅處理 auto-repair success / approval EXECUTION_SUCCESS / approval EXPIRED,
|
||||
# 不自動關閉 manual_required 或單純 APPROVED 事件。
|
||||
# API production pool 只有 serving 預算;實際 loop 由單一 standalone
|
||||
# signal worker 擁有,避免 coroutine 被 budget gate 關閉後仍假性記錄
|
||||
# scheduled。reconciler 本身只處理強證據或已不再 firing 的事件。
|
||||
try:
|
||||
from src.jobs.incident_lifecycle_reconciler import (
|
||||
INTERVAL_SECONDS as INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
|
||||
)
|
||||
from src.jobs.incident_lifecycle_reconciler import (
|
||||
run_incident_lifecycle_reconciler_loop,
|
||||
)
|
||||
schedule_api_background_task(run_incident_lifecycle_reconciler_loop())
|
||||
logger.info(
|
||||
"incident_lifecycle_reconciler_scheduled",
|
||||
"incident_lifecycle_reconciler_owner_delegated",
|
||||
interval_sec=INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
|
||||
scheduled_in_api_process=False,
|
||||
production_process_owner="awoooi-worker",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("incident_lifecycle_reconciler_schedule_failed", error=str(e))
|
||||
logger.warning(
|
||||
"incident_lifecycle_reconciler_owner_resolution_failed",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
# AwoooP Ansible candidate backfill worker.
|
||||
# 把近期已命中 allowlisted PlayBook、但缺 durable candidate row 的事故補進
|
||||
|
||||
@@ -703,13 +703,14 @@ def _inspect_source_contract(repo_root: Path) -> dict[str, Any]:
|
||||
|
||||
|
||||
def _read_source_contract_text(repo_root: Path, relative_path: str) -> str:
|
||||
candidates = [repo_root / relative_path]
|
||||
source_path = repo_root / relative_path
|
||||
candidates = [source_path]
|
||||
api_prefix = "apps/api/"
|
||||
if relative_path.startswith(api_prefix):
|
||||
candidates.append(repo_root / relative_path.removeprefix(api_prefix))
|
||||
if (
|
||||
relative_path == _AWOOOP_VERIFIED_CONTEXT_SURFACE_SOURCE
|
||||
and not (repo_root / "apps").exists()
|
||||
and not source_path.exists()
|
||||
):
|
||||
candidates.append(repo_root / "src/services/telegram_alert_ai_automation_matrix.py")
|
||||
|
||||
|
||||
@@ -623,7 +623,35 @@ async def _main() -> None:
|
||||
backup_restore_legacy_backfill_task: asyncio.Task[Any] | None = None
|
||||
security_maintenance_tasks: list[asyncio.Task[Any]] = []
|
||||
agent99_controlled_dispatch_reconciler_task: asyncio.Task[Any] | None = None
|
||||
incident_lifecycle_reconciler_task: asyncio.Task[Any] | None = None
|
||||
mcp_version_lifecycle_task: asyncio.Task[Any] | None = None
|
||||
|
||||
# Production API pods reserve their DB pool for serving requests. This
|
||||
# singleton worker therefore owns the existing bounded, fail-closed
|
||||
# lifecycle reconciler. It resolves at most one configured batch per pass
|
||||
# and independently reads current Prometheus firing state before treating
|
||||
# inactive alerts as closure candidates.
|
||||
from src.jobs.incident_lifecycle_reconciler import (
|
||||
BATCH_LIMIT as INCIDENT_LIFECYCLE_RECONCILER_BATCH_LIMIT,
|
||||
)
|
||||
from src.jobs.incident_lifecycle_reconciler import (
|
||||
INTERVAL_SECONDS as INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
|
||||
)
|
||||
from src.jobs.incident_lifecycle_reconciler import (
|
||||
run_incident_lifecycle_reconciler_loop,
|
||||
)
|
||||
|
||||
incident_lifecycle_reconciler_task = asyncio.create_task(
|
||||
run_incident_lifecycle_reconciler_loop(),
|
||||
name="run_incident_lifecycle_reconciler_loop",
|
||||
)
|
||||
logger.info(
|
||||
"signal_worker_incident_lifecycle_reconciler_started",
|
||||
owner="signal_worker",
|
||||
interval_seconds=INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
|
||||
batch_limit=INCIDENT_LIFECYCLE_RECONCILER_BATCH_LIMIT,
|
||||
direct_incident_closure_performed=False,
|
||||
)
|
||||
if settings.ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER:
|
||||
from src.jobs.awooop_ansible_candidate_backfill_job import (
|
||||
run_awooop_ansible_candidate_backfill_loop,
|
||||
@@ -794,6 +822,12 @@ async def _main() -> None:
|
||||
await agent99_controlled_dispatch_reconciler_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if incident_lifecycle_reconciler_task is not None:
|
||||
incident_lifecycle_reconciler_task.cancel()
|
||||
try:
|
||||
await incident_lifecycle_reconciler_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if mcp_version_lifecycle_task is not None:
|
||||
mcp_version_lifecycle_task.cancel()
|
||||
try:
|
||||
|
||||
@@ -539,6 +539,40 @@ def test_signal_worker_gracefully_cancels_agent99_reconciler() -> None:
|
||||
assert source.index(await_task) < source.index("await worker.stop()")
|
||||
|
||||
|
||||
def test_signal_worker_is_single_incident_lifecycle_reconciler_owner() -> None:
|
||||
import inspect
|
||||
|
||||
from src import main as api_main
|
||||
from src.workers import signal_worker
|
||||
|
||||
api_source = inspect.getsource(api_main.lifespan)
|
||||
worker_source = inspect.getsource(signal_worker._main)
|
||||
|
||||
assert "run_incident_lifecycle_reconciler_loop" not in api_source
|
||||
assert "incident_lifecycle_reconciler_owner_delegated" in api_source
|
||||
assert worker_source.count("run_incident_lifecycle_reconciler_loop()") == 1
|
||||
assert worker_source.index("if settings.DATABASE_BOOTSTRAP_DDL_ENABLED") < (
|
||||
worker_source.index("run_incident_lifecycle_reconciler_loop()")
|
||||
)
|
||||
assert "signal_worker_incident_lifecycle_reconciler_started" in worker_source
|
||||
assert 'owner="signal_worker"' in worker_source
|
||||
|
||||
|
||||
def test_signal_worker_gracefully_cancels_incident_lifecycle_reconciler() -> None:
|
||||
import inspect
|
||||
|
||||
from src.workers import signal_worker
|
||||
|
||||
source = inspect.getsource(signal_worker._main)
|
||||
cancel = "incident_lifecycle_reconciler_task.cancel()"
|
||||
await_task = "await incident_lifecycle_reconciler_task"
|
||||
|
||||
assert cancel in source
|
||||
assert await_task in source
|
||||
assert source.index(cancel) < source.index(await_task)
|
||||
assert source.index(await_task) < source.index("await worker.stop()")
|
||||
|
||||
|
||||
def test_production_signal_worker_is_single_agent99_reconciler_owner() -> None:
|
||||
import inspect
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from fastapi.testclient import TestClient
|
||||
from src.api.v1.agents import router
|
||||
from src.services.telegram_alert_ai_automation_matrix import (
|
||||
_inspect_source_contract,
|
||||
_read_source_contract_text,
|
||||
load_latest_telegram_alert_ai_automation_matrix,
|
||||
)
|
||||
|
||||
@@ -210,6 +211,28 @@ def test_telegram_matrix_source_contract_supports_container_layout(tmp_path):
|
||||
assert proof["source_contract_missing_markers"] == []
|
||||
|
||||
|
||||
def test_telegram_matrix_source_contract_uses_embedded_surface_when_web_package_only(
|
||||
tmp_path,
|
||||
):
|
||||
(tmp_path / "apps/web").mkdir(parents=True)
|
||||
(tmp_path / "apps/web/package.json").write_text("{}\n", encoding="utf-8")
|
||||
(tmp_path / "src/services").mkdir(parents=True)
|
||||
embedded_source = (
|
||||
tmp_path / "src/services/telegram_alert_ai_automation_matrix.py"
|
||||
)
|
||||
embedded_source.write_text(
|
||||
'data-testid="telegram-alert-post-apply-verifier-readback"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
source = _read_source_contract_text(
|
||||
tmp_path,
|
||||
"apps/web/src/components/awooop/autonomous-runtime-receipt-panel.tsx",
|
||||
)
|
||||
|
||||
assert 'data-testid="telegram-alert-post-apply-verifier-readback"' in source
|
||||
|
||||
|
||||
def test_telegram_matrix_source_contract_reports_missing_web_surface_without_raising(
|
||||
tmp_path,
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user