121 lines
4.7 KiB
Python
121 lines
4.7 KiB
Python
"""Scheduler adapter for the decision-only Nemotron runtime canary."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Callable
|
|
|
|
|
|
def run_scheduled_nemotron_decision_canary(
|
|
*,
|
|
env_flag: Callable[[str, bool], bool],
|
|
save_stats: Callable[[str, dict[str, object]], None],
|
|
notify_failure: Callable[..., None],
|
|
) -> dict[str, object]:
|
|
"""Run one model decision, persist proof, and send lifecycle acknowledgement."""
|
|
if not env_flag("NEMOTRON_DECISION_CANARY_SCHEDULED_ENABLED", True):
|
|
payload: dict[str, object] = {
|
|
"success": True,
|
|
"status": "skipped",
|
|
"terminal_status": "no_write_terminal",
|
|
"reason": "scheduled_canary_disabled_by_policy",
|
|
"next_machine_action": "enable_nemotron_decision_canary_after_policy_review",
|
|
}
|
|
save_stats("nemotron_decision_canary", payload)
|
|
return payload
|
|
|
|
try:
|
|
from services.nemotron_decision_canary_service import (
|
|
acknowledge_nemotron_decision_canary,
|
|
run_nemotron_decision_canary,
|
|
)
|
|
from services.telegram_templates import send_telegram_with_result
|
|
|
|
payload = run_nemotron_decision_canary(
|
|
execute=True,
|
|
write_receipt=True,
|
|
)
|
|
execution = payload.get("execution") or {}
|
|
telegram = send_telegram_with_result(
|
|
"\n".join([
|
|
"Nemotron decision-only canary",
|
|
f"status: {payload.get('status')}",
|
|
f"host: {execution.get('selected_host')}",
|
|
f"model: {execution.get('model')}",
|
|
f"decision: {execution.get('decision_tool')}",
|
|
f"elapsed_ms: {execution.get('model_elapsed_ms')}",
|
|
"side effects: tool=0, database=0, Telegram action=0",
|
|
f"run_id: {(payload.get('run_identity') or {}).get('run_id')}",
|
|
]),
|
|
parse_mode=None,
|
|
)
|
|
telegram_status = "acknowledged" if telegram.get("ok") else "failed"
|
|
receipt_path = str(execution.get("receipt_path") or "")
|
|
receipt_acknowledged = False
|
|
if receipt_path:
|
|
acknowledged = acknowledge_nemotron_decision_canary(
|
|
receipt_path,
|
|
telegram_status=telegram_status,
|
|
telegram_sent=int(telegram.get("sent") or 0),
|
|
telegram_failed=int(telegram.get("failed") or 0),
|
|
)
|
|
payload["execution"] = acknowledged
|
|
payload["latest_execution"] = run_nemotron_decision_canary(
|
|
execute=False
|
|
).get("latest_execution") or {}
|
|
receipt_acknowledged = True
|
|
else:
|
|
payload["receipt_error"] = "durable_execution_receipt_missing"
|
|
|
|
verified = bool(
|
|
payload.get("success")
|
|
and telegram_status == "acknowledged"
|
|
and receipt_acknowledged
|
|
)
|
|
payload["terminal_status"] = (
|
|
"verified_decision_only_no_write" if verified else "partial"
|
|
)
|
|
save_stats("nemotron_decision_canary", payload)
|
|
logging.info(
|
|
"[NemotronCanary] status=%s terminal=%s model_ms=%s telegram=%s receipt=%s",
|
|
payload.get("status"),
|
|
payload.get("terminal_status"),
|
|
execution.get("model_elapsed_ms"),
|
|
telegram_status,
|
|
"acknowledged" if receipt_acknowledged else "missing",
|
|
)
|
|
if not verified:
|
|
notify_failure(
|
|
"run_nemotron_decision_canary_task",
|
|
RuntimeError(
|
|
"Nemotron decision canary closure or acknowledgement failed"
|
|
),
|
|
source="Scheduler.NemotronCanary",
|
|
event_type="nemotron_decision_canary_failure",
|
|
title="Nemotron decision-only canary failed",
|
|
dedup_ttl_sec=86400,
|
|
)
|
|
return payload
|
|
except Exception as error:
|
|
logging.error("[NemotronCanary] task failed: %s", error, exc_info=True)
|
|
notify_failure(
|
|
"run_nemotron_decision_canary_task",
|
|
error,
|
|
source="Scheduler.NemotronCanary",
|
|
event_type="nemotron_decision_canary_task_failure",
|
|
title="Nemotron decision-only canary task failed",
|
|
dedup_ttl_sec=86400,
|
|
)
|
|
payload = {
|
|
"success": False,
|
|
"status": "failed",
|
|
"terminal_status": "partial",
|
|
"error": f"{type(error).__name__}: {str(error)[:300]}",
|
|
"next_machine_action": "inspect_nemotron_canary_receipt_and_retry",
|
|
}
|
|
save_stats("nemotron_decision_canary", payload)
|
|
return payload
|
|
|
|
|
|
__all__ = ["run_scheduled_nemotron_decision_canary"]
|