Some checks failed
Code Review / ai-code-review (push) Successful in 13s
CD Pipeline / tests (push) Failing after 1m8s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Ansible / Reboot Recovery Contract / validate (push) Has been cancelled
2522 lines
94 KiB
Python
2522 lines
94 KiB
Python
"""AwoooP inbound channel event dossier service.
|
|
|
|
T15c: converts redacted inbound source envelopes into an Operator Console DTO.
|
|
The service is read-only and does not mutate incident, run, approval, or
|
|
automation state.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any, Literal
|
|
from uuid import UUID
|
|
|
|
import structlog
|
|
from fastapi import HTTPException, status
|
|
from sqlalchemy import text
|
|
|
|
from src.db.base import get_db_context
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
_MAX_DOSSIER_EVENTS = 50
|
|
_MAX_COVERAGE_EVENTS = 200
|
|
_MAX_RECURRENCE_EVENTS = 300
|
|
_MAX_REPAIR_INCIDENTS = 200
|
|
_SOURCE_CORRELATION_REVIEW_PROVIDERS = {"sentry", "signoz"}
|
|
_SOURCE_CORRELATION_REVIEW_EXCLUDED_STAGES = {"heartbeat"}
|
|
_SOURCE_CORRELATION_WORK_ITEM_ID_MAX = 180
|
|
_SOURCE_CORRELATION_DECISION_SCHEMA_VERSION = (
|
|
"awooop_source_correlation_review_decision_v1"
|
|
)
|
|
_SOURCE_CORRELATION_APPLY_SCHEMA_VERSION = "awooop_source_correlation_apply_v1"
|
|
_SOURCE_CORRELATION_APPLY_STAGE = "source_correlation_linked"
|
|
_SOURCE_CORRELATION_REVIEW_ACTOR = "awooop_source_correlation_review_service"
|
|
_SOURCE_CORRELATION_APPLY_ACTOR = "awooop_source_correlation_apply_service"
|
|
_INCIDENT_ID_RE = re.compile(r"\bINC-\d{8}-[A-Z0-9]{4,}\b")
|
|
RecurrenceWorkItemMode = Literal["auto", "ticket", "reverify", "approval_review", "observe"]
|
|
RecurrenceWorkItemHandoffKind = Literal["ticket_proposal", "manual_review"]
|
|
SourceCorrelationReviewDecision = Literal[
|
|
"accepted",
|
|
"rejected",
|
|
"needs_more_evidence",
|
|
]
|
|
|
|
|
|
class RecurrenceWorkItemNotFoundError(LookupError):
|
|
"""Requested recurrence work item is not in the current read model."""
|
|
|
|
|
|
def _as_dict(value: Any) -> dict[str, Any]:
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def _json_safe(value: Any) -> Any:
|
|
if isinstance(value, dict):
|
|
return {str(key): _json_safe(item) for key, item in value.items()}
|
|
if isinstance(value, list | tuple | set):
|
|
return [_json_safe(item) for item in value]
|
|
if value is None or isinstance(value, str | int | float | bool):
|
|
return value
|
|
return str(value)
|
|
|
|
|
|
def _compact_ref_count(source_refs: dict[str, Any]) -> int:
|
|
total = 0
|
|
for value in source_refs.values():
|
|
if isinstance(value, list):
|
|
total += len(value)
|
|
elif value:
|
|
total += 1
|
|
return total
|
|
|
|
|
|
def _ref_count(source_refs: dict[str, Any], key: str) -> int:
|
|
value = source_refs.get(key)
|
|
if isinstance(value, list):
|
|
return len(value)
|
|
return 1 if value else 0
|
|
|
|
|
|
def _source_correlation_ref_total(group: dict[str, Any]) -> int:
|
|
return int(group.get("sentry_ref_total") or 0) + int(
|
|
group.get("signoz_ref_total") or 0
|
|
)
|
|
|
|
|
|
def _needs_source_correlation_review(
|
|
group: dict[str, Any],
|
|
latest_incident_id: str | None,
|
|
) -> bool:
|
|
if latest_incident_id:
|
|
return False
|
|
provider = str(group.get("provider") or "").lower()
|
|
stage = str(group.get("latest_stage") or "").lower()
|
|
if provider not in _SOURCE_CORRELATION_REVIEW_PROVIDERS:
|
|
return False
|
|
if stage in _SOURCE_CORRELATION_REVIEW_EXCLUDED_STAGES:
|
|
return False
|
|
return _source_correlation_ref_total(group) > 0
|
|
|
|
|
|
def _source_correlation_work_item_id(group: dict[str, Any]) -> str:
|
|
source_id = str(
|
|
group.get("latest_provider_event_id") or group.get("recurrence_key") or "unknown"
|
|
).strip()
|
|
return f"source-evidence:{source_id}"[:_SOURCE_CORRELATION_WORK_ITEM_ID_MAX]
|
|
|
|
|
|
def _source_correlation_review_outcome(decision: str) -> dict[str, str]:
|
|
if decision == "accepted":
|
|
return {
|
|
"status": "source_correlation_accepted",
|
|
"next_step": "verify_source_match_in_status_chain",
|
|
"reason": "provider_native_evidence_accepted",
|
|
"review_status": "accepted",
|
|
}
|
|
if decision == "rejected":
|
|
return {
|
|
"status": "source_correlation_rejected",
|
|
"next_step": "monitor_for_new_provider_evidence",
|
|
"reason": "provider_native_evidence_rejected",
|
|
"review_status": "rejected",
|
|
}
|
|
return {
|
|
"status": "source_correlation_review",
|
|
"next_step": "collect_more_source_evidence",
|
|
"reason": "provider_native_evidence_needs_more_evidence",
|
|
"review_status": "needs_more_evidence",
|
|
}
|
|
|
|
|
|
def _normalize_source_review_decision(row: dict[str, Any]) -> dict[str, Any] | None:
|
|
context = _as_dict(row.get("context"))
|
|
if context.get("schema_version") != _SOURCE_CORRELATION_DECISION_SCHEMA_VERSION:
|
|
return None
|
|
work_item_id = str(context.get("work_item_id") or "").strip()
|
|
decision = str(context.get("decision") or "").strip()
|
|
if not work_item_id or decision not in {
|
|
"accepted",
|
|
"rejected",
|
|
"needs_more_evidence",
|
|
}:
|
|
return None
|
|
|
|
return {
|
|
"schema_version": _SOURCE_CORRELATION_DECISION_SCHEMA_VERSION,
|
|
"review_id": row.get("id"),
|
|
"work_item_id": work_item_id,
|
|
"decision": decision,
|
|
"review_status": context.get("review_status") or decision,
|
|
"target_incident_id": context.get("target_incident_id"),
|
|
"reviewer_id": context.get("reviewer_id"),
|
|
"operator_note": context.get("operator_note"),
|
|
"latest_provider_event_id": context.get("latest_provider_event_id"),
|
|
"latest_stage": context.get("latest_stage"),
|
|
"recorded_at": row.get("created_at"),
|
|
"history": context.get("history") or {},
|
|
}
|
|
|
|
|
|
def _normalize_source_apply(row: dict[str, Any]) -> dict[str, Any] | None:
|
|
context = _as_dict(row.get("context"))
|
|
if context.get("schema_version") != _SOURCE_CORRELATION_APPLY_SCHEMA_VERSION:
|
|
return None
|
|
work_item_id = str(context.get("work_item_id") or "").strip()
|
|
target_incident_id = str(context.get("target_incident_id") or "").strip()
|
|
if not work_item_id or not target_incident_id:
|
|
return None
|
|
|
|
history = _as_dict(context.get("history"))
|
|
return {
|
|
"schema_version": _SOURCE_CORRELATION_APPLY_SCHEMA_VERSION,
|
|
"apply_id": row.get("id"),
|
|
"work_item_id": work_item_id,
|
|
"apply_status": context.get("apply_status") or "applied",
|
|
"target_incident_id": target_incident_id,
|
|
"review_id": context.get("review_id"),
|
|
"reviewer_id": context.get("reviewer_id"),
|
|
"operator_note": context.get("operator_note"),
|
|
"latest_provider_event_id": context.get("latest_provider_event_id"),
|
|
"source_event_provider_event_id": context.get(
|
|
"source_event_provider_event_id"
|
|
),
|
|
"source_event_id": history.get("source_event_id"),
|
|
"timeline_event_id": history.get("timeline_event_id"),
|
|
"recorded_at": row.get("created_at"),
|
|
"history": history,
|
|
}
|
|
|
|
|
|
def _provider_raw_event_id(provider_event_id: Any) -> str:
|
|
parts = str(provider_event_id or "").split(":", 2)
|
|
if len(parts) == 3 and parts[2].strip():
|
|
return parts[2].strip()
|
|
return str(provider_event_id or "unknown").strip() or "unknown"
|
|
|
|
|
|
def _append_unique(values: list[str], candidate: Any) -> None:
|
|
text_value = str(candidate or "").strip()
|
|
if text_value and text_value not in values:
|
|
values.append(text_value)
|
|
|
|
|
|
def _append_incident_ids_from_text(values: list[str], text_value: Any) -> None:
|
|
if not text_value:
|
|
return
|
|
for incident_id in _INCIDENT_ID_RE.findall(str(text_value)):
|
|
_append_unique(values, incident_id)
|
|
|
|
|
|
def _append_incident_ids_from_refs(
|
|
values: list[str], source_refs: dict[str, Any]
|
|
) -> None:
|
|
incident_ids = source_refs.get("incident_ids")
|
|
if isinstance(incident_ids, list):
|
|
for incident_id in incident_ids:
|
|
_append_unique(values, incident_id)
|
|
else:
|
|
_append_unique(values, incident_ids)
|
|
|
|
|
|
def _event_incident_ids(event: dict[str, Any]) -> list[str]:
|
|
incident_ids: list[str] = []
|
|
_append_incident_ids_from_refs(incident_ids, _as_dict(event.get("source_refs")))
|
|
_append_incident_ids_from_text(incident_ids, event.get("content_preview"))
|
|
_append_incident_ids_from_text(incident_ids, event.get("content_redacted"))
|
|
_append_incident_ids_from_text(incident_ids, event.get("provider_event_id"))
|
|
return incident_ids
|
|
|
|
|
|
def _recurrence_key(event: dict[str, Any]) -> str:
|
|
fingerprint = str(event.get("fingerprint") or "").strip()
|
|
if fingerprint:
|
|
return f"fingerprint:{fingerprint}"
|
|
|
|
provider = str(event.get("provider") or event.get("channel_type") or "unknown")
|
|
alertname = str(event.get("alertname") or "").strip()
|
|
namespace = str(event.get("namespace") or "").strip()
|
|
target = str(event.get("target_resource") or "").strip()
|
|
if alertname or namespace or target:
|
|
return f"alert:{provider}:{alertname}:{namespace}:{target}"
|
|
|
|
return f"event:{provider}:{event.get('provider_event_id')}"
|
|
|
|
|
|
def build_dossier_recurrence(
|
|
rows: list[dict[str, Any]],
|
|
*,
|
|
project_id: str,
|
|
limit: int,
|
|
repair_summaries_by_incident: dict[str, dict[str, Any]] | None = None,
|
|
source_review_decisions_by_work_item: dict[str, dict[str, Any]] | None = None,
|
|
source_applies_by_work_item: dict[str, dict[str, Any]] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Group recent source events into recurrence buckets with linked run state."""
|
|
groups: dict[str, dict[str, Any]] = {}
|
|
repair_summaries = repair_summaries_by_incident or {}
|
|
source_review_decisions = source_review_decisions_by_work_item or {}
|
|
source_applies = source_applies_by_work_item or {}
|
|
|
|
for row in rows:
|
|
event = build_dossier_event(row)
|
|
key = _recurrence_key(event)
|
|
source_ref_count = int(event.get("source_ref_count") or 0)
|
|
source_refs = _as_dict(event.get("source_refs"))
|
|
incident_ids = _event_incident_ids(event)
|
|
run_id = row.get("run_id")
|
|
run_state = row.get("run_state")
|
|
received_at = event.get("received_at")
|
|
|
|
group = groups.setdefault(
|
|
key,
|
|
{
|
|
"recurrence_key": key,
|
|
"provider": event.get("provider"),
|
|
"alertname": event.get("alertname"),
|
|
"severity": event.get("severity"),
|
|
"namespace": event.get("namespace"),
|
|
"target_resource": event.get("target_resource"),
|
|
"fingerprint": event.get("fingerprint"),
|
|
"latest_stage": event.get("stage"),
|
|
"latest_event_id": event.get("event_id"),
|
|
"latest_provider_event_id": event.get("provider_event_id"),
|
|
"latest_content_preview": event.get("content_preview"),
|
|
"latest_run_id": run_id,
|
|
"latest_run_state": run_state,
|
|
"latest_agent_id": row.get("run_agent_id"),
|
|
"latest_incident_id": incident_ids[0] if incident_ids else None,
|
|
"incident_ids": [],
|
|
"occurrence_total": 0,
|
|
"duplicate_total": 0,
|
|
"linked_run_total": 0,
|
|
"source_ref_total": 0,
|
|
"missing_source_refs_total": 0,
|
|
"sentry_ref_total": 0,
|
|
"signoz_ref_total": 0,
|
|
"alert_ref_total": 0,
|
|
"stage_counts": {},
|
|
"run_state_counts": {},
|
|
"first_received_at": received_at,
|
|
"latest_received_at": received_at,
|
|
"_run_ids": set(),
|
|
"_source_correlation_work_item_ids": [],
|
|
},
|
|
)
|
|
|
|
group["occurrence_total"] += 1
|
|
group["source_ref_total"] += source_ref_count
|
|
if source_ref_count <= 0:
|
|
group["missing_source_refs_total"] += 1
|
|
if event.get("is_duplicate"):
|
|
group["duplicate_total"] += 1
|
|
|
|
stage = str(event.get("stage") or "received")
|
|
stage_counts = group["stage_counts"]
|
|
stage_counts[stage] = int(stage_counts.get(stage, 0)) + 1
|
|
if (
|
|
str(event.get("provider") or "").lower()
|
|
in _SOURCE_CORRELATION_REVIEW_PROVIDERS
|
|
and stage.lower() not in _SOURCE_CORRELATION_REVIEW_EXCLUDED_STAGES
|
|
and source_ref_count > 0
|
|
and not incident_ids
|
|
):
|
|
_append_unique(
|
|
group["_source_correlation_work_item_ids"],
|
|
_source_correlation_work_item_id({
|
|
"latest_provider_event_id": event.get("provider_event_id"),
|
|
"recurrence_key": key,
|
|
}),
|
|
)
|
|
|
|
for incident_id in incident_ids:
|
|
_append_unique(group["incident_ids"], incident_id)
|
|
|
|
group["sentry_ref_total"] += _ref_count(source_refs, "sentry_issue_ids")
|
|
group["signoz_ref_total"] += _ref_count(source_refs, "signoz_alerts")
|
|
group["alert_ref_total"] += _ref_count(source_refs, "alert_ids")
|
|
|
|
if run_id:
|
|
group["_run_ids"].add(str(run_id))
|
|
if group.get("latest_run_id") is None:
|
|
group["latest_run_id"] = run_id
|
|
group["latest_run_state"] = run_state
|
|
group["latest_agent_id"] = row.get("run_agent_id")
|
|
if run_state:
|
|
state_counts = group["run_state_counts"]
|
|
state_counts[str(run_state)] = int(state_counts.get(str(run_state), 0)) + 1
|
|
|
|
if received_at and (
|
|
group.get("first_received_at") is None
|
|
or str(received_at) < str(group.get("first_received_at"))
|
|
):
|
|
group["first_received_at"] = received_at
|
|
if received_at and (
|
|
group.get("latest_received_at") is None
|
|
or str(received_at) > str(group.get("latest_received_at"))
|
|
):
|
|
group["latest_received_at"] = received_at
|
|
group["latest_event_id"] = event.get("event_id")
|
|
group["latest_provider_event_id"] = event.get("provider_event_id")
|
|
group["latest_content_preview"] = event.get("content_preview")
|
|
group["latest_stage"] = event.get("stage")
|
|
group["latest_incident_id"] = (
|
|
incident_ids[0] if incident_ids else group.get("latest_incident_id")
|
|
)
|
|
|
|
items = []
|
|
linked_run_total = 0
|
|
for group in groups.values():
|
|
run_ids = group.pop("_run_ids")
|
|
group["linked_run_total"] = len(run_ids)
|
|
_attach_work_item_summary(
|
|
group,
|
|
repair_summaries,
|
|
source_review_decisions,
|
|
source_applies,
|
|
)
|
|
linked_run_total += len(run_ids)
|
|
items.append(group)
|
|
|
|
items.sort(key=lambda item: str(item.get("latest_received_at") or ""), reverse=True)
|
|
items.sort(key=lambda item: int(item.get("occurrence_total") or 0), reverse=True)
|
|
latest_received_at = max(
|
|
(
|
|
item.get("latest_received_at")
|
|
for item in items
|
|
if item.get("latest_received_at")
|
|
),
|
|
default=None,
|
|
)
|
|
|
|
return {
|
|
"project_id": project_id,
|
|
"limit": limit,
|
|
"summary": {
|
|
"source_event_total": len(rows),
|
|
"recurrence_group_total": len(items),
|
|
"recurrent_group_total": sum(
|
|
1 for item in items if int(item.get("occurrence_total") or 0) > 1
|
|
),
|
|
"duplicate_event_total": sum(
|
|
int(item.get("duplicate_total") or 0) for item in items
|
|
),
|
|
"linked_run_total": linked_run_total,
|
|
"unlinked_event_total": sum(1 for row in rows if not row.get("run_id")),
|
|
"auto_repair_linked_total": sum(
|
|
1
|
|
for item in items
|
|
if _as_dict(item.get("repair_summary")).get("latest_auto_repair_id")
|
|
),
|
|
"verified_repair_group_total": sum(
|
|
1
|
|
for item in items
|
|
if _as_dict(item.get("repair_summary")).get("status")
|
|
== "auto_repair_verified"
|
|
),
|
|
"open_work_item_group_total": sum(
|
|
1
|
|
for item in items
|
|
if _as_dict(item.get("work_item")).get("status") == "open"
|
|
),
|
|
"manual_gate_group_total": sum(
|
|
1
|
|
for item in items
|
|
if _as_dict(item.get("repair_summary")).get("status") == "manual_gate"
|
|
),
|
|
"controlled_apply_gate_group_total": sum(
|
|
1
|
|
for item in items
|
|
if _as_dict(item.get("repair_summary")).get("status")
|
|
== "controlled_apply_gate"
|
|
),
|
|
"automation_gap_group_total": sum(
|
|
1
|
|
for item in items
|
|
if _as_dict(item.get("repair_summary")).get("status")
|
|
== "run_completed_no_repair"
|
|
),
|
|
"failed_repair_group_total": sum(
|
|
1
|
|
for item in items
|
|
if _as_dict(item.get("repair_summary")).get("status")
|
|
== "auto_repair_failed"
|
|
),
|
|
"source_correlation_review_group_total": sum(
|
|
1
|
|
for item in items
|
|
if _as_dict(item.get("repair_summary")).get("status")
|
|
== "source_correlation_review"
|
|
),
|
|
"source_correlation_decision_recorded_group_total": sum(
|
|
1 for item in items if item.get("source_correlation_review")
|
|
),
|
|
"source_correlation_applied_group_total": sum(
|
|
1 for item in items if item.get("source_correlation_apply")
|
|
),
|
|
"latest_received_at": latest_received_at,
|
|
},
|
|
"items": items,
|
|
}
|
|
|
|
|
|
def _repair_status(
|
|
*,
|
|
incident_id: str | None,
|
|
latest_run_state: str | None,
|
|
repair_summary: dict[str, Any] | None,
|
|
) -> str:
|
|
if not incident_id:
|
|
return "no_incident_link"
|
|
if repair_summary:
|
|
latest_success = repair_summary.get("latest_success")
|
|
verification = str(
|
|
repair_summary.get("latest_verification_result") or ""
|
|
).lower()
|
|
if latest_success is True and verification == "success":
|
|
return "auto_repair_verified"
|
|
if latest_success is True:
|
|
return "auto_repair_succeeded_unverified"
|
|
if latest_success is False:
|
|
return "auto_repair_failed"
|
|
return "auto_repair_recorded"
|
|
if latest_run_state == "waiting_approval":
|
|
return "controlled_apply_gate"
|
|
if latest_run_state in {"pending", "running", "waiting_tool"}:
|
|
return "investigating"
|
|
if latest_run_state == "completed":
|
|
return "run_completed_no_repair"
|
|
return "no_repair_record"
|
|
|
|
|
|
def _work_item_status(repair_status: str) -> str:
|
|
if repair_status == "no_incident_link":
|
|
return "none"
|
|
if repair_status in {
|
|
"auto_repair_verified",
|
|
"source_correlation_accepted",
|
|
"source_correlation_rejected",
|
|
}:
|
|
return "closed"
|
|
return "open"
|
|
|
|
|
|
def _work_item_kind(repair_status: str, auto_repair_id: Any) -> str:
|
|
if repair_status in {
|
|
"source_correlation_review",
|
|
"source_correlation_accepted",
|
|
"source_correlation_rejected",
|
|
}:
|
|
return "source_correlation_review"
|
|
if auto_repair_id:
|
|
return "verification"
|
|
if repair_status == "run_completed_no_repair":
|
|
return "automation_gap"
|
|
if repair_status == "controlled_apply_gate":
|
|
return "controlled_apply_followup"
|
|
if repair_status == "investigating":
|
|
return "investigation"
|
|
return "incident_followup"
|
|
|
|
|
|
def _work_item_next_step(repair_status: str) -> str:
|
|
return {
|
|
"source_correlation_review": "review_provider_source_match",
|
|
"source_correlation_accepted": "verify_source_match_in_status_chain",
|
|
"source_correlation_rejected": "monitor_for_new_provider_evidence",
|
|
"source_correlation_applied": "verify_source_link_in_status_chain",
|
|
"auto_repair_succeeded_unverified": "run_post_verification",
|
|
"auto_repair_failed": "triage_failed_repair",
|
|
"auto_repair_recorded": "review_repair_record",
|
|
"controlled_apply_gate": "evaluate_controlled_apply",
|
|
"manual_gate": "legacy_manual_gate_migrated_to_controlled_apply",
|
|
"investigating": "wait_for_run_completion",
|
|
"run_completed_no_repair": "create_repair_ticket",
|
|
"no_repair_record": "triage_missing_repair_record",
|
|
}.get(repair_status, "none")
|
|
|
|
|
|
def _work_item_reason(repair_status: str) -> str:
|
|
return {
|
|
"source_correlation_review": "provider_native_evidence_unlinked",
|
|
"source_correlation_accepted": "provider_native_evidence_accepted",
|
|
"source_correlation_rejected": "provider_native_evidence_rejected",
|
|
"source_correlation_applied": "provider_native_evidence_link_applied",
|
|
"auto_repair_succeeded_unverified": "auto_repair_missing_verification",
|
|
"auto_repair_failed": "auto_repair_failed",
|
|
"auto_repair_recorded": "auto_repair_record_needs_review",
|
|
"controlled_apply_gate": "controlled_apply_required",
|
|
"manual_gate": "legacy_manual_gate_migrated_to_controlled_apply",
|
|
"investigating": "run_still_investigating",
|
|
"run_completed_no_repair": "completed_run_without_auto_repair",
|
|
"no_repair_record": "incident_without_repair_record",
|
|
}.get(repair_status, "none")
|
|
|
|
|
|
def _attach_work_item_summary(
|
|
group: dict[str, Any],
|
|
repair_summaries_by_incident: dict[str, dict[str, Any]],
|
|
source_review_decisions_by_work_item: dict[str, dict[str, Any]],
|
|
source_applies_by_work_item: dict[str, dict[str, Any]],
|
|
) -> None:
|
|
incident_ids = [
|
|
str(incident_id) for incident_id in group.get("incident_ids", []) if incident_id
|
|
]
|
|
latest_incident_id = str(group.get("latest_incident_id") or "") or (
|
|
incident_ids[0] if incident_ids else None
|
|
)
|
|
repair_summary = (
|
|
repair_summaries_by_incident.get(latest_incident_id)
|
|
if latest_incident_id
|
|
else None
|
|
)
|
|
status_value = _repair_status(
|
|
incident_id=latest_incident_id,
|
|
latest_run_state=group.get("latest_run_state"),
|
|
repair_summary=repair_summary,
|
|
)
|
|
if _needs_source_correlation_review(group, latest_incident_id):
|
|
status_value = "source_correlation_review"
|
|
|
|
if repair_summary:
|
|
repair_payload = dict(repair_summary)
|
|
repair_payload["status"] = status_value
|
|
else:
|
|
repair_payload = {
|
|
"schema_version": "awooop_recurrence_repair_summary_v1",
|
|
"status": status_value,
|
|
"incident_id": latest_incident_id,
|
|
"latest_auto_repair_id": None,
|
|
"latest_verification_result": None,
|
|
"auto_repair_total": 0,
|
|
"success_total": 0,
|
|
"failed_total": 0,
|
|
}
|
|
|
|
work_status = _work_item_status(status_value)
|
|
auto_repair_id = repair_payload.get("latest_auto_repair_id")
|
|
work_item_id = None
|
|
if latest_incident_id and work_status != "none":
|
|
work_item_id = (
|
|
f"verification:{latest_incident_id}:{auto_repair_id}"
|
|
if auto_repair_id
|
|
else f"incident:{latest_incident_id}"
|
|
)
|
|
elif status_value == "source_correlation_review" and work_status != "none":
|
|
work_item_id = _source_correlation_work_item_id(group)
|
|
|
|
source_work_item_ids = [
|
|
str(candidate)
|
|
for candidate in group.pop("_source_correlation_work_item_ids", [])
|
|
if candidate
|
|
]
|
|
candidate_work_item_ids = [
|
|
candidate
|
|
for candidate in [work_item_id, *source_work_item_ids]
|
|
if candidate
|
|
]
|
|
source_review_decision = next(
|
|
(
|
|
source_review_decisions_by_work_item[candidate]
|
|
for candidate in candidate_work_item_ids
|
|
if candidate in source_review_decisions_by_work_item
|
|
),
|
|
None,
|
|
)
|
|
source_apply = next(
|
|
(
|
|
source_applies_by_work_item[candidate]
|
|
for candidate in candidate_work_item_ids
|
|
if candidate in source_applies_by_work_item
|
|
),
|
|
None,
|
|
)
|
|
matched_incident_id = None
|
|
work_item_next_step = _work_item_next_step(status_value)
|
|
work_item_reason = _work_item_reason(status_value)
|
|
if source_review_decision:
|
|
outcome = _source_correlation_review_outcome(
|
|
str(source_review_decision.get("decision") or "")
|
|
)
|
|
status_value = outcome["status"]
|
|
repair_payload["status"] = status_value
|
|
work_status = _work_item_status(status_value)
|
|
matched_incident_id = source_review_decision.get("target_incident_id")
|
|
work_item_next_step = outcome["next_step"]
|
|
work_item_reason = outcome["reason"]
|
|
group["source_correlation_review"] = source_review_decision
|
|
if source_apply:
|
|
matched_incident_id = source_apply.get("target_incident_id")
|
|
group["source_correlation_apply"] = source_apply
|
|
if status_value in {
|
|
"source_correlation_review",
|
|
"source_correlation_accepted",
|
|
}:
|
|
work_item_next_step = "verify_source_link_in_status_chain"
|
|
work_item_reason = "provider_native_evidence_link_applied"
|
|
|
|
group["latest_incident_id"] = latest_incident_id
|
|
group["repair_summary"] = repair_payload
|
|
group["work_item"] = {
|
|
"schema_version": "awooop_recurrence_work_item_link_v1",
|
|
"work_item_id": work_item_id,
|
|
"incident_id": latest_incident_id,
|
|
"matched_incident_id": matched_incident_id,
|
|
"auto_repair_id": auto_repair_id,
|
|
"status": work_status,
|
|
"kind": _work_item_kind(status_value, auto_repair_id),
|
|
"next_step": work_item_next_step,
|
|
"reason": work_item_reason,
|
|
"needs_human": work_status == "open",
|
|
}
|
|
|
|
|
|
def _recurrence_work_item_target(item: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"recurrence_key": item.get("recurrence_key"),
|
|
"provider": item.get("provider"),
|
|
"alertname": item.get("alertname"),
|
|
"severity": item.get("severity"),
|
|
"namespace": item.get("namespace"),
|
|
"target_resource": item.get("target_resource"),
|
|
"fingerprint": item.get("fingerprint"),
|
|
"latest_stage": item.get("latest_stage"),
|
|
"latest_event_id": item.get("latest_event_id"),
|
|
"latest_provider_event_id": item.get("latest_provider_event_id"),
|
|
"latest_run_id": item.get("latest_run_id"),
|
|
"latest_run_state": item.get("latest_run_state"),
|
|
"latest_agent_id": item.get("latest_agent_id"),
|
|
"latest_incident_id": item.get("latest_incident_id"),
|
|
"matched_incident_id": _as_dict(item.get("work_item")).get(
|
|
"matched_incident_id"
|
|
),
|
|
}
|
|
|
|
|
|
def _selected_recurrence_mode(
|
|
work_item: dict[str, Any],
|
|
requested_mode: RecurrenceWorkItemMode,
|
|
) -> str:
|
|
if requested_mode != "auto":
|
|
return requested_mode
|
|
|
|
next_step = str(work_item.get("next_step") or "")
|
|
if next_step == "create_repair_ticket":
|
|
return "ticket"
|
|
if next_step in {
|
|
"run_post_verification",
|
|
"triage_failed_repair",
|
|
"review_repair_record",
|
|
"triage_missing_repair_record",
|
|
}:
|
|
return "reverify"
|
|
if next_step == "review_approval":
|
|
return "approval_review"
|
|
return "observe"
|
|
|
|
|
|
def _recurrence_work_item_checks(
|
|
item: dict[str, Any],
|
|
work_item: dict[str, Any],
|
|
) -> list[dict[str, Any]]:
|
|
repair_summary = _as_dict(item.get("repair_summary"))
|
|
source_ref_total = int(item.get("source_ref_total") or 0)
|
|
is_source_review = work_item.get("kind") == "source_correlation_review"
|
|
evidence_linked = bool(item.get("latest_provider_event_id")) and source_ref_total > 0
|
|
incident_or_source_linked = bool(work_item.get("incident_id")) or (
|
|
is_source_review and evidence_linked
|
|
)
|
|
return [
|
|
{
|
|
"name": "work_item_open",
|
|
"passed": work_item.get("status") == "open",
|
|
"detail": str(work_item.get("status") or "unknown"),
|
|
},
|
|
{
|
|
"name": "incident_or_source_evidence_linked",
|
|
"passed": incident_or_source_linked,
|
|
"detail": str(
|
|
work_item.get("incident_id")
|
|
or item.get("latest_provider_event_id")
|
|
or "missing incident_id/source evidence"
|
|
),
|
|
},
|
|
{
|
|
"name": "known_next_step",
|
|
"passed": str(work_item.get("next_step") or "none") != "none",
|
|
"detail": str(work_item.get("next_step") or "none"),
|
|
},
|
|
{
|
|
"name": "source_refs_present",
|
|
"passed": source_ref_total > 0,
|
|
"detail": str(source_ref_total),
|
|
},
|
|
{
|
|
"name": "no_destructive_writes",
|
|
"passed": True,
|
|
"detail": "preview_and_dry_run_only",
|
|
},
|
|
{
|
|
"name": "repair_status_visible",
|
|
"passed": bool(repair_summary.get("status")),
|
|
"detail": str(repair_summary.get("status") or "unknown"),
|
|
},
|
|
]
|
|
|
|
|
|
def _recurrence_plan(
|
|
item: dict[str, Any],
|
|
work_item: dict[str, Any],
|
|
mode: str,
|
|
) -> dict[str, Any]:
|
|
route_by_mode = {
|
|
"ticket": {
|
|
"step": "prepare_repair_ticket_preview",
|
|
"flywheel_node": "work_item_to_ticket",
|
|
"agent_id": "awooop_recurrence_coordinator",
|
|
},
|
|
"reverify": {
|
|
"step": "collect_read_model_and_prepare_reverification",
|
|
"flywheel_node": "verify",
|
|
"agent_id": "post_execution_verifier",
|
|
},
|
|
"approval_review": {
|
|
"step": "route_to_approval_review",
|
|
"flywheel_node": "approval",
|
|
"agent_id": "awooop_approval_coordinator",
|
|
},
|
|
"observe": {
|
|
"step": "observe_until_run_or_repair_state_changes",
|
|
"flywheel_node": "observe",
|
|
"agent_id": "awooop_recurrence_coordinator",
|
|
},
|
|
}
|
|
route = route_by_mode.get(mode, route_by_mode["observe"])
|
|
return {
|
|
**route,
|
|
"required_scope": "read",
|
|
"writes": [],
|
|
"target_action": work_item.get("next_step"),
|
|
"reason": work_item.get("reason"),
|
|
"target": _recurrence_work_item_target(item),
|
|
}
|
|
|
|
|
|
def _ticket_preview(item: dict[str, Any], work_item: dict[str, Any]) -> dict[str, Any]:
|
|
alertname = str(item.get("alertname") or item.get("provider") or "recurrence")
|
|
incident_id = str(work_item.get("incident_id") or item.get("latest_incident_id") or "")
|
|
kind = str(work_item.get("kind") or "recurrence")
|
|
if kind == "source_correlation_review":
|
|
title = f"[AwoooP] Source evidence review: {alertname}"
|
|
labels = ["awooop", "source-correlation", "review", str(item.get("provider") or "source")]
|
|
body_lines = [
|
|
f"Provider event: {item.get('latest_provider_event_id') or '--'}",
|
|
f"Stage: {item.get('latest_stage') or '--'}",
|
|
f"Provider: {item.get('provider') or '--'}",
|
|
f"Alert: {alertname}",
|
|
f"Namespace/Target: {item.get('namespace') or '--'} / {item.get('target_resource') or '--'}",
|
|
f"Source refs: {item.get('source_ref_total') or 0}",
|
|
f"Sentry refs: {item.get('sentry_ref_total') or 0}",
|
|
f"SignOz refs: {item.get('signoz_ref_total') or 0}",
|
|
f"Next step: {work_item.get('next_step') or '--'}",
|
|
"Writes: none in preview/dry-run; source matching requires a later explicit review/apply path.",
|
|
]
|
|
else:
|
|
title = f"[AwoooP] {alertname} recurrence work item: {incident_id or 'unlinked'}"
|
|
labels = ["awooop", "recurrence", kind]
|
|
body_lines = [
|
|
f"Incident: {incident_id or '--'}",
|
|
f"Alert: {alertname}",
|
|
f"Namespace/Target: {item.get('namespace') or '--'} / {item.get('target_resource') or '--'}",
|
|
f"Occurrences: {item.get('occurrence_total') or 0}",
|
|
f"Duplicates: {item.get('duplicate_total') or 0}",
|
|
f"Latest run: {item.get('latest_run_id') or '--'} ({item.get('latest_run_state') or '--'})",
|
|
f"Repair status: {_as_dict(item.get('repair_summary')).get('status') or '--'}",
|
|
f"Next step: {work_item.get('next_step') or '--'}",
|
|
"Writes: none in preview/dry-run; ticket creation requires a later explicit apply path.",
|
|
]
|
|
return {
|
|
"would_create": False,
|
|
"title": title[:180],
|
|
"labels": labels,
|
|
"body_preview": "\n".join(body_lines)[:1000],
|
|
}
|
|
|
|
|
|
def _recurrence_current_state_summary(
|
|
item: dict[str, Any],
|
|
work_item: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
repair_summary = _as_dict(item.get("repair_summary"))
|
|
return {
|
|
"work_item_status": work_item.get("status"),
|
|
"work_item_kind": work_item.get("kind"),
|
|
"work_item_next_step": work_item.get("next_step"),
|
|
"work_item_reason": work_item.get("reason"),
|
|
"occurrence_total": int(item.get("occurrence_total") or 0),
|
|
"duplicate_total": int(item.get("duplicate_total") or 0),
|
|
"linked_run_total": int(item.get("linked_run_total") or 0),
|
|
"run_state_counts": item.get("run_state_counts") or {},
|
|
"stage_counts": item.get("stage_counts") or {},
|
|
"latest_stage": item.get("latest_stage"),
|
|
"latest_provider_event_id": item.get("latest_provider_event_id"),
|
|
"latest_run_state": item.get("latest_run_state"),
|
|
"latest_run_id": item.get("latest_run_id"),
|
|
"matched_incident_id": work_item.get("matched_incident_id"),
|
|
"source_correlation_review": item.get("source_correlation_review"),
|
|
"source_correlation_apply": item.get("source_correlation_apply"),
|
|
"repair_status": repair_summary.get("status"),
|
|
"latest_auto_repair_id": repair_summary.get("latest_auto_repair_id"),
|
|
"latest_verification_result": repair_summary.get("latest_verification_result"),
|
|
"auto_repair_total": int(repair_summary.get("auto_repair_total") or 0),
|
|
"source_ref_total": int(item.get("source_ref_total") or 0),
|
|
"sentry_ref_total": int(item.get("sentry_ref_total") or 0),
|
|
"signoz_ref_total": int(item.get("signoz_ref_total") or 0),
|
|
"alert_ref_total": int(item.get("alert_ref_total") or 0),
|
|
}
|
|
|
|
|
|
def _verification_result_preview(mode: str, allowed: bool) -> str:
|
|
if not allowed:
|
|
return "blocked"
|
|
return {
|
|
"ticket": "ticket_preview_ready",
|
|
"reverify": "reverify_preview_ready",
|
|
"approval_review": "approval_review_required",
|
|
"observe": "observe_only",
|
|
}.get(mode, "observe_only")
|
|
|
|
|
|
def _find_recurrence_work_item(
|
|
recurrence: dict[str, Any],
|
|
work_item_id: str,
|
|
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
for item in recurrence.get("items") or []:
|
|
work_item = _as_dict(item.get("work_item"))
|
|
if work_item.get("work_item_id") == work_item_id:
|
|
return item, work_item
|
|
raise RecurrenceWorkItemNotFoundError(work_item_id)
|
|
|
|
|
|
def build_recurrence_work_item_preview(
|
|
recurrence: dict[str, Any],
|
|
*,
|
|
work_item_id: str,
|
|
mode: RecurrenceWorkItemMode = "auto",
|
|
) -> dict[str, Any]:
|
|
"""Build a read-only plan for a recurrence work item."""
|
|
|
|
item, work_item = _find_recurrence_work_item(recurrence, work_item_id)
|
|
selected_mode = _selected_recurrence_mode(work_item, mode)
|
|
checks = _recurrence_work_item_checks(item, work_item)
|
|
allowed = all(check["passed"] for check in checks)
|
|
|
|
return {
|
|
"schema_version": "awooop_recurrence_work_item_preview_v1",
|
|
"source": "channel_event_dossier.recurrence",
|
|
"project_id": recurrence.get("project_id"),
|
|
"work_item_id": work_item.get("work_item_id"),
|
|
"incident_id": work_item.get("incident_id"),
|
|
"auto_repair_id": work_item.get("auto_repair_id"),
|
|
"mode": selected_mode,
|
|
"requested_mode": mode,
|
|
"allowed": allowed,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"checks": checks,
|
|
"plan": _recurrence_plan(item, work_item, selected_mode),
|
|
}
|
|
|
|
|
|
def build_recurrence_work_item_dry_run(
|
|
recurrence: dict[str, Any],
|
|
*,
|
|
work_item_id: str,
|
|
mode: RecurrenceWorkItemMode = "auto",
|
|
) -> dict[str, Any]:
|
|
"""Build a read-only dry-run result for a recurrence work item."""
|
|
|
|
item, work_item = _find_recurrence_work_item(recurrence, work_item_id)
|
|
selected_mode = _selected_recurrence_mode(work_item, mode)
|
|
checks = _recurrence_work_item_checks(item, work_item)
|
|
allowed = all(check["passed"] for check in checks)
|
|
payload = {
|
|
"schema_version": "awooop_recurrence_work_item_dry_run_v1",
|
|
"source": "channel_event_dossier.recurrence",
|
|
"project_id": recurrence.get("project_id"),
|
|
"work_item_id": work_item.get("work_item_id"),
|
|
"incident_id": work_item.get("incident_id"),
|
|
"auto_repair_id": work_item.get("auto_repair_id"),
|
|
"mode": selected_mode,
|
|
"requested_mode": mode,
|
|
"allowed": allowed,
|
|
"executed": allowed,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"checks": checks,
|
|
"verification_result_preview": _verification_result_preview(
|
|
selected_mode,
|
|
allowed,
|
|
),
|
|
"current_state_summary": _recurrence_current_state_summary(item, work_item),
|
|
"ticket_preview": _ticket_preview(item, work_item),
|
|
"plan": _recurrence_plan(item, work_item, selected_mode),
|
|
"read_model_route": {
|
|
"agent_id": "awooop_recurrence_coordinator",
|
|
"tool_name": "channel_event_dossier.recurrence",
|
|
"required_scope": "read",
|
|
"is_shadow": True,
|
|
"flywheel_node": "work_item",
|
|
},
|
|
"next_step": work_item.get("next_step"),
|
|
}
|
|
if not allowed:
|
|
payload["executed"] = False
|
|
return payload
|
|
|
|
|
|
def _recurrence_handoff_next_step(
|
|
handoff_kind: RecurrenceWorkItemHandoffKind,
|
|
allowed: bool,
|
|
) -> str:
|
|
if not allowed:
|
|
return "fix_preflight_checks"
|
|
if handoff_kind == "manual_review":
|
|
return "operator_manual_review"
|
|
return "operator_review_ticket_preview"
|
|
|
|
|
|
def build_recurrence_work_item_handoff(
|
|
recurrence: dict[str, Any],
|
|
*,
|
|
work_item_id: str,
|
|
mode: RecurrenceWorkItemMode = "auto",
|
|
handoff_kind: RecurrenceWorkItemHandoffKind = "ticket_proposal",
|
|
) -> dict[str, Any]:
|
|
"""Build a record-only handoff proposal for a recurrence work item."""
|
|
|
|
payload = build_recurrence_work_item_dry_run(
|
|
recurrence,
|
|
work_item_id=work_item_id,
|
|
mode=mode,
|
|
)
|
|
allowed = bool(payload.get("allowed"))
|
|
plan = _as_dict(payload.get("plan"))
|
|
read_model_route = _as_dict(payload.get("read_model_route"))
|
|
payload.update(
|
|
{
|
|
"schema_version": "awooop_recurrence_work_item_handoff_v1",
|
|
"handoff_kind": handoff_kind,
|
|
"handoff_status": "ready_to_record" if allowed else "blocked",
|
|
"handoff_owner": "operator",
|
|
"safety_level": "handoff_record_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"creates_external_ticket": False,
|
|
"plan": {
|
|
**plan,
|
|
"step": "record_recurrence_work_item_handoff",
|
|
"flywheel_node": "handoff",
|
|
"required_scope": "record_history",
|
|
"writes": ["timeline_events", "alert_operation_log"],
|
|
},
|
|
"read_model_route": {
|
|
**read_model_route,
|
|
"required_scope": "record_history",
|
|
"is_shadow": False,
|
|
"flywheel_node": "handoff",
|
|
},
|
|
"next_step": _recurrence_handoff_next_step(handoff_kind, allowed),
|
|
}
|
|
)
|
|
return payload
|
|
|
|
|
|
def _recurrence_history_context(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "awooop_recurrence_work_item_dry_run_history_v1",
|
|
"source": payload.get("source"),
|
|
"project_id": payload.get("project_id"),
|
|
"work_item_id": payload.get("work_item_id"),
|
|
"incident_id": payload.get("incident_id"),
|
|
"auto_repair_id": payload.get("auto_repair_id"),
|
|
"mode": payload.get("mode"),
|
|
"requested_mode": payload.get("requested_mode"),
|
|
"allowed": payload.get("allowed"),
|
|
"executed": payload.get("executed"),
|
|
"safety_level": payload.get("safety_level"),
|
|
"writes_incident_state": payload.get("writes_incident_state"),
|
|
"writes_auto_repair_result": payload.get("writes_auto_repair_result"),
|
|
"writes_ticket": payload.get("writes_ticket"),
|
|
"verification_result_preview": payload.get("verification_result_preview"),
|
|
"current_state_summary": payload.get("current_state_summary"),
|
|
"ticket_preview": payload.get("ticket_preview"),
|
|
"read_model_route": payload.get("read_model_route"),
|
|
"checks": payload.get("checks"),
|
|
"next_step": payload.get("next_step"),
|
|
}
|
|
|
|
|
|
def _recurrence_handoff_context(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "awooop_recurrence_work_item_handoff_history_v1",
|
|
"source": payload.get("source"),
|
|
"project_id": payload.get("project_id"),
|
|
"work_item_id": payload.get("work_item_id"),
|
|
"incident_id": payload.get("incident_id"),
|
|
"auto_repair_id": payload.get("auto_repair_id"),
|
|
"mode": payload.get("mode"),
|
|
"requested_mode": payload.get("requested_mode"),
|
|
"handoff_kind": payload.get("handoff_kind"),
|
|
"handoff_status": payload.get("handoff_status"),
|
|
"handoff_owner": payload.get("handoff_owner"),
|
|
"allowed": payload.get("allowed"),
|
|
"executed": payload.get("executed"),
|
|
"safety_level": payload.get("safety_level"),
|
|
"writes_incident_state": payload.get("writes_incident_state"),
|
|
"writes_auto_repair_result": payload.get("writes_auto_repair_result"),
|
|
"writes_ticket": payload.get("writes_ticket"),
|
|
"creates_external_ticket": payload.get("creates_external_ticket"),
|
|
"verification_result_preview": payload.get("verification_result_preview"),
|
|
"current_state_summary": payload.get("current_state_summary"),
|
|
"ticket_preview": payload.get("ticket_preview"),
|
|
"read_model_route": payload.get("read_model_route"),
|
|
"checks": payload.get("checks"),
|
|
"next_step": payload.get("next_step"),
|
|
}
|
|
|
|
|
|
def _is_source_correlation_review_payload(payload: dict[str, Any]) -> bool:
|
|
state = _as_dict(payload.get("current_state_summary"))
|
|
return (
|
|
state.get("work_item_kind") == "source_correlation_review"
|
|
or payload.get("next_step") == "review_provider_source_match"
|
|
)
|
|
|
|
|
|
async def _record_recurrence_work_item_dry_run_history(
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
incident_id = str(payload.get("incident_id") or "")
|
|
source_review = _is_source_correlation_review_payload(payload)
|
|
if not incident_id and not source_review:
|
|
return {"recorded": False, "reason": "missing_incident_id"}
|
|
|
|
history: dict[str, Any] = {
|
|
"recorded": False,
|
|
"alert_operation_id": None,
|
|
"timeline_event_id": None,
|
|
}
|
|
context = _recurrence_history_context(payload)
|
|
allowed = bool(payload.get("allowed"))
|
|
|
|
try:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
record = await get_alert_operation_log_repository().append(
|
|
"PRE_FLIGHT_PASSED" if allowed else "PRE_FLIGHT_FAILED",
|
|
incident_id=incident_id or None,
|
|
auto_repair_id=str(payload.get("auto_repair_id") or "") or None,
|
|
actor="awooop_recurrence_work_item_service",
|
|
action_detail=f"recurrence_work_item_dry_run:{payload.get('mode')}"[:200],
|
|
success=allowed,
|
|
context=_json_safe(context),
|
|
)
|
|
if record is not None:
|
|
history["alert_operation_id"] = getattr(record, "id", None)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"awooop_recurrence_work_item_alert_operation_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
if incident_id:
|
|
try:
|
|
from src.services.approval_db import get_timeline_service
|
|
|
|
event = await get_timeline_service().add_event(
|
|
event_type="verifier",
|
|
status="success" if allowed else "warning",
|
|
title="AwoooP recurrence work item dry-run",
|
|
description=_recurrence_history_description(context),
|
|
actor="awooop_recurrence_work_item_service",
|
|
actor_role=str(payload.get("mode") or "dry_run"),
|
|
incident_id=incident_id,
|
|
)
|
|
if event:
|
|
history["timeline_event_id"] = event.get("id")
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"awooop_recurrence_work_item_timeline_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
else:
|
|
history["timeline_reason"] = "source_review_not_incident_scoped"
|
|
|
|
history["recorded"] = bool(
|
|
history.get("alert_operation_id") or history.get("timeline_event_id")
|
|
)
|
|
if not history["recorded"]:
|
|
history["reason"] = "history_sink_unavailable"
|
|
return history
|
|
|
|
|
|
async def _record_recurrence_work_item_handoff_history(
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
incident_id = str(payload.get("incident_id") or "")
|
|
source_review = _is_source_correlation_review_payload(payload)
|
|
if not incident_id and not source_review:
|
|
return {"recorded": False, "reason": "missing_incident_id"}
|
|
|
|
history: dict[str, Any] = {
|
|
"recorded": False,
|
|
"alert_operation_id": None,
|
|
"timeline_event_id": None,
|
|
}
|
|
context = _recurrence_handoff_context(payload)
|
|
allowed = bool(payload.get("allowed"))
|
|
|
|
try:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
record = await get_alert_operation_log_repository().append(
|
|
"ESCALATED",
|
|
incident_id=incident_id or None,
|
|
auto_repair_id=str(payload.get("auto_repair_id") or "") or None,
|
|
actor="awooop_recurrence_work_item_service",
|
|
action_detail=(
|
|
f"recurrence_work_item_handoff:{payload.get('handoff_kind')}"
|
|
)[:200],
|
|
success=allowed,
|
|
context=_json_safe(context),
|
|
)
|
|
if record is not None:
|
|
history["alert_operation_id"] = getattr(record, "id", None)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"awooop_recurrence_work_item_handoff_alert_operation_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
if incident_id:
|
|
try:
|
|
from src.services.approval_db import get_timeline_service
|
|
|
|
event = await get_timeline_service().add_event(
|
|
event_type="human",
|
|
status="warning" if allowed else "error",
|
|
title="AwoooP recurrence work item handoff",
|
|
description=_recurrence_handoff_history_description(context),
|
|
actor="awooop_recurrence_work_item_service",
|
|
actor_role=str(payload.get("handoff_kind") or "handoff"),
|
|
incident_id=incident_id,
|
|
)
|
|
if event:
|
|
history["timeline_event_id"] = event.get("id")
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"awooop_recurrence_work_item_handoff_timeline_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
else:
|
|
history["timeline_reason"] = "source_review_not_incident_scoped"
|
|
|
|
history["recorded"] = bool(
|
|
history.get("alert_operation_id") or history.get("timeline_event_id")
|
|
)
|
|
if not history["recorded"]:
|
|
history["reason"] = "history_sink_unavailable"
|
|
return history
|
|
|
|
|
|
def _recurrence_history_description(context: dict[str, Any]) -> str:
|
|
state = context.get("current_state_summary") or {}
|
|
route = context.get("read_model_route") or {}
|
|
return (
|
|
f"mode={context.get('mode')} "
|
|
f"preview={context.get('verification_result_preview')} "
|
|
f"occurrences={state.get('occurrence_total')} "
|
|
f"repair_status={state.get('repair_status')} "
|
|
f"route={route.get('agent_id')}/{route.get('tool_name')} "
|
|
f"writes_incident={context.get('writes_incident_state')} "
|
|
f"writes_auto_repair={context.get('writes_auto_repair_result')} "
|
|
f"writes_ticket={context.get('writes_ticket')}"
|
|
)[:500]
|
|
|
|
|
|
def _recurrence_handoff_history_description(context: dict[str, Any]) -> str:
|
|
state = context.get("current_state_summary") or {}
|
|
route = context.get("read_model_route") or {}
|
|
return (
|
|
f"handoff={context.get('handoff_kind')} "
|
|
f"status={context.get('handoff_status')} "
|
|
f"mode={context.get('mode')} "
|
|
f"occurrences={state.get('occurrence_total')} "
|
|
f"repair_status={state.get('repair_status')} "
|
|
f"route={route.get('agent_id')}/{route.get('tool_name')} "
|
|
f"external_ticket={context.get('creates_external_ticket')} "
|
|
f"writes_incident={context.get('writes_incident_state')} "
|
|
f"writes_auto_repair={context.get('writes_auto_repair_result')} "
|
|
f"writes_ticket={context.get('writes_ticket')}"
|
|
)[:500]
|
|
|
|
|
|
def _source_correlation_review_checks(
|
|
item: dict[str, Any],
|
|
work_item: dict[str, Any],
|
|
*,
|
|
decision: str,
|
|
target_incident_id: str | None,
|
|
) -> list[dict[str, Any]]:
|
|
source_ref_total = int(item.get("source_ref_total") or 0)
|
|
is_source_review = work_item.get("kind") == "source_correlation_review"
|
|
checks = [
|
|
{
|
|
"name": "source_review_work_item",
|
|
"passed": is_source_review,
|
|
"detail": str(work_item.get("kind") or "unknown"),
|
|
},
|
|
{
|
|
"name": "known_decision",
|
|
"passed": decision
|
|
in {"accepted", "rejected", "needs_more_evidence"},
|
|
"detail": decision or "unknown",
|
|
},
|
|
{
|
|
"name": "source_refs_present",
|
|
"passed": source_ref_total > 0,
|
|
"detail": str(source_ref_total),
|
|
},
|
|
{
|
|
"name": "provider_event_present",
|
|
"passed": bool(item.get("latest_provider_event_id")),
|
|
"detail": str(item.get("latest_provider_event_id") or "missing"),
|
|
},
|
|
{
|
|
"name": "review_record_only",
|
|
"passed": True,
|
|
"detail": "alert_operation_log_only",
|
|
},
|
|
]
|
|
if decision == "accepted":
|
|
checks.append(
|
|
{
|
|
"name": "target_incident_required",
|
|
"passed": bool(
|
|
target_incident_id and _INCIDENT_ID_RE.search(target_incident_id)
|
|
),
|
|
"detail": target_incident_id or "missing target_incident_id",
|
|
}
|
|
)
|
|
return checks
|
|
|
|
|
|
def _source_correlation_review_context(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": _SOURCE_CORRELATION_DECISION_SCHEMA_VERSION,
|
|
"source": payload.get("source"),
|
|
"project_id": payload.get("project_id"),
|
|
"work_item_id": payload.get("work_item_id"),
|
|
"decision": payload.get("decision"),
|
|
"review_status": payload.get("review_status"),
|
|
"target_incident_id": payload.get("target_incident_id"),
|
|
"reviewer_id": payload.get("reviewer_id"),
|
|
"operator_note": payload.get("operator_note"),
|
|
"latest_provider_event_id": payload.get("latest_provider_event_id"),
|
|
"latest_stage": payload.get("latest_stage"),
|
|
"provider": payload.get("provider"),
|
|
"alertname": payload.get("alertname"),
|
|
"namespace": payload.get("namespace"),
|
|
"target_resource": payload.get("target_resource"),
|
|
"safety_level": payload.get("safety_level"),
|
|
"writes_incident_state": payload.get("writes_incident_state"),
|
|
"writes_source_event": payload.get("writes_source_event"),
|
|
"writes_auto_repair_result": payload.get("writes_auto_repair_result"),
|
|
"writes_ticket": payload.get("writes_ticket"),
|
|
"creates_external_ticket": payload.get("creates_external_ticket"),
|
|
"checks": payload.get("checks"),
|
|
"current_state_summary": payload.get("current_state_summary"),
|
|
"plan": payload.get("plan"),
|
|
"read_model_route": payload.get("read_model_route"),
|
|
"next_step": payload.get("next_step"),
|
|
"history": payload.get("history"),
|
|
}
|
|
|
|
|
|
def _source_correlation_review_description(context: dict[str, Any]) -> str:
|
|
return (
|
|
f"decision={context.get('decision')} "
|
|
f"work_item={context.get('work_item_id')} "
|
|
f"provider_event={context.get('latest_provider_event_id')} "
|
|
f"target_incident={context.get('target_incident_id') or '--'} "
|
|
f"writes_incident={context.get('writes_incident_state')} "
|
|
f"writes_source_event={context.get('writes_source_event')}"
|
|
)[:500]
|
|
|
|
|
|
def build_source_correlation_review_decision(
|
|
recurrence: dict[str, Any],
|
|
*,
|
|
work_item_id: str,
|
|
decision: SourceCorrelationReviewDecision,
|
|
target_incident_id: str | None = None,
|
|
reviewer_id: str = "operator_console",
|
|
operator_note: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Build a record-only source-correlation review decision payload."""
|
|
|
|
item, work_item = _find_recurrence_work_item(recurrence, work_item_id)
|
|
safe_decision = str(decision)
|
|
target_id = str(target_incident_id or "").strip() or None
|
|
checks = _source_correlation_review_checks(
|
|
item,
|
|
work_item,
|
|
decision=safe_decision,
|
|
target_incident_id=target_id,
|
|
)
|
|
allowed = all(check["passed"] for check in checks)
|
|
outcome = _source_correlation_review_outcome(safe_decision)
|
|
writes_timeline = bool(target_id and safe_decision == "accepted")
|
|
|
|
payload = {
|
|
"schema_version": _SOURCE_CORRELATION_DECISION_SCHEMA_VERSION,
|
|
"source": "channel_event_dossier.recurrence",
|
|
"project_id": recurrence.get("project_id"),
|
|
"work_item_id": work_item.get("work_item_id"),
|
|
"incident_id": work_item.get("incident_id"),
|
|
"target_incident_id": target_id,
|
|
"decision": safe_decision,
|
|
"review_status": outcome["review_status"],
|
|
"reviewer_id": str(reviewer_id or "operator_console")[:100],
|
|
"operator_note": str(operator_note or "").strip()[:500] or None,
|
|
"allowed": allowed,
|
|
"executed": allowed,
|
|
"safety_level": "source_review_record_only",
|
|
"writes_incident_state": False,
|
|
"writes_source_event": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"creates_external_ticket": False,
|
|
"latest_provider_event_id": item.get("latest_provider_event_id"),
|
|
"latest_stage": item.get("latest_stage"),
|
|
"provider": item.get("provider"),
|
|
"alertname": item.get("alertname"),
|
|
"namespace": item.get("namespace"),
|
|
"target_resource": item.get("target_resource"),
|
|
"checks": checks,
|
|
"current_state_summary": _recurrence_current_state_summary(item, work_item),
|
|
"plan": {
|
|
"step": "record_source_correlation_review_decision",
|
|
"flywheel_node": "source_correlation_review",
|
|
"agent_id": "awooop_source_correlation_reviewer",
|
|
"required_scope": "record_history",
|
|
"writes": ["alert_operation_log", *(["timeline_events"] if writes_timeline else [])],
|
|
"target_action": outcome["next_step"],
|
|
"reason": outcome["reason"],
|
|
"target": _recurrence_work_item_target(item),
|
|
},
|
|
"read_model_route": {
|
|
"agent_id": "awooop_source_correlation_reviewer",
|
|
"tool_name": "channel_event_dossier.recurrence",
|
|
"required_scope": "record_history",
|
|
"is_shadow": False,
|
|
"flywheel_node": "source_correlation_review",
|
|
},
|
|
"next_step": outcome["next_step"] if allowed else "fix_preflight_checks",
|
|
}
|
|
if not allowed:
|
|
payload["executed"] = False
|
|
return payload
|
|
|
|
|
|
async def _record_source_correlation_review_decision_history(
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if not payload.get("allowed"):
|
|
return {"recorded": False, "reason": "preflight_failed"}
|
|
|
|
incident_id = str(payload.get("target_incident_id") or "") or None
|
|
history: dict[str, Any] = {
|
|
"recorded": False,
|
|
"alert_operation_id": None,
|
|
"timeline_event_id": None,
|
|
}
|
|
context = _source_correlation_review_context(payload)
|
|
|
|
try:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
record = await get_alert_operation_log_repository().append(
|
|
"USER_ACTION",
|
|
incident_id=incident_id,
|
|
actor=_SOURCE_CORRELATION_REVIEW_ACTOR,
|
|
action_detail=f"source_correlation_review:{payload.get('decision')}"[:200],
|
|
success=True,
|
|
context=_json_safe(context),
|
|
)
|
|
if record is not None:
|
|
history["alert_operation_id"] = getattr(record, "id", None)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"awooop_source_correlation_review_alert_operation_failed",
|
|
work_item_id=payload.get("work_item_id"),
|
|
error=str(exc),
|
|
)
|
|
|
|
if incident_id and payload.get("decision") == "accepted":
|
|
try:
|
|
from src.services.approval_db import get_timeline_service
|
|
|
|
event = await get_timeline_service().add_event(
|
|
event_type="human",
|
|
status="success",
|
|
title="AwoooP source correlation review",
|
|
description=_source_correlation_review_description(context),
|
|
actor=_SOURCE_CORRELATION_REVIEW_ACTOR,
|
|
actor_role="source_correlation_review",
|
|
incident_id=incident_id,
|
|
)
|
|
if event:
|
|
history["timeline_event_id"] = event.get("id")
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"awooop_source_correlation_review_timeline_failed",
|
|
incident_id=incident_id,
|
|
work_item_id=payload.get("work_item_id"),
|
|
error=str(exc),
|
|
)
|
|
else:
|
|
history["timeline_reason"] = "review_not_incident_timeline_scoped"
|
|
|
|
history["recorded"] = bool(
|
|
history.get("alert_operation_id") or history.get("timeline_event_id")
|
|
)
|
|
if not history["recorded"]:
|
|
history["reason"] = "history_sink_unavailable"
|
|
return history
|
|
|
|
|
|
def _source_correlation_apply_checks(
|
|
item: dict[str, Any],
|
|
work_item: dict[str, Any],
|
|
source_review: dict[str, Any],
|
|
*,
|
|
target_incident_id: str | None,
|
|
) -> list[dict[str, Any]]:
|
|
provider = str(item.get("provider") or "").lower()
|
|
source_ref_total = int(item.get("source_ref_total") or 0)
|
|
return [
|
|
{
|
|
"name": "source_review_work_item",
|
|
"passed": work_item.get("kind") == "source_correlation_review",
|
|
"detail": str(work_item.get("kind") or "unknown"),
|
|
},
|
|
{
|
|
"name": "accepted_review_recorded",
|
|
"passed": source_review.get("decision") == "accepted",
|
|
"detail": str(source_review.get("decision") or "missing"),
|
|
},
|
|
{
|
|
"name": "target_incident_present",
|
|
"passed": bool(target_incident_id),
|
|
"detail": target_incident_id or "missing target_incident_id",
|
|
},
|
|
{
|
|
"name": "provider_supported",
|
|
"passed": provider in _SOURCE_CORRELATION_REVIEW_PROVIDERS,
|
|
"detail": provider or "unknown",
|
|
},
|
|
{
|
|
"name": "source_refs_present",
|
|
"passed": source_ref_total > 0,
|
|
"detail": str(source_ref_total),
|
|
},
|
|
{
|
|
"name": "provider_event_present",
|
|
"passed": bool(item.get("latest_provider_event_id")),
|
|
"detail": str(item.get("latest_provider_event_id") or "missing"),
|
|
},
|
|
{
|
|
"name": "append_only_source_link",
|
|
"passed": True,
|
|
"detail": "awooop_conversation_event_append_only",
|
|
},
|
|
]
|
|
|
|
|
|
def _source_correlation_apply_context(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": _SOURCE_CORRELATION_APPLY_SCHEMA_VERSION,
|
|
"source": payload.get("source"),
|
|
"project_id": payload.get("project_id"),
|
|
"work_item_id": payload.get("work_item_id"),
|
|
"review_id": payload.get("review_id"),
|
|
"apply_status": payload.get("apply_status"),
|
|
"target_incident_id": payload.get("target_incident_id"),
|
|
"reviewer_id": payload.get("reviewer_id"),
|
|
"operator_note": payload.get("operator_note"),
|
|
"latest_provider_event_id": payload.get("latest_provider_event_id"),
|
|
"source_event_provider_event_id": payload.get(
|
|
"source_event_provider_event_id"
|
|
),
|
|
"source_event_stage": payload.get("source_event_stage"),
|
|
"provider": payload.get("provider"),
|
|
"alertname": payload.get("alertname"),
|
|
"namespace": payload.get("namespace"),
|
|
"target_resource": payload.get("target_resource"),
|
|
"safety_level": payload.get("safety_level"),
|
|
"writes_incident_state": payload.get("writes_incident_state"),
|
|
"writes_source_event": payload.get("writes_source_event"),
|
|
"writes_auto_repair_result": payload.get("writes_auto_repair_result"),
|
|
"writes_ticket": payload.get("writes_ticket"),
|
|
"creates_external_ticket": payload.get("creates_external_ticket"),
|
|
"checks": payload.get("checks"),
|
|
"current_state_summary": payload.get("current_state_summary"),
|
|
"plan": payload.get("plan"),
|
|
"read_model_route": payload.get("read_model_route"),
|
|
"next_step": payload.get("next_step"),
|
|
"history": payload.get("history"),
|
|
}
|
|
|
|
|
|
def _source_correlation_apply_description(context: dict[str, Any]) -> str:
|
|
return (
|
|
f"work_item={context.get('work_item_id')} "
|
|
f"provider_event={context.get('latest_provider_event_id')} "
|
|
f"source_event={context.get('source_event_provider_event_id')} "
|
|
f"target_incident={context.get('target_incident_id')} "
|
|
f"writes_source_event={context.get('writes_source_event')} "
|
|
f"writes_incident={context.get('writes_incident_state')}"
|
|
)[:500]
|
|
|
|
|
|
def build_source_correlation_apply(
|
|
recurrence: dict[str, Any],
|
|
*,
|
|
work_item_id: str,
|
|
reviewer_id: str = "operator_console",
|
|
operator_note: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Build an append-only source link apply payload for an accepted review."""
|
|
|
|
item, work_item = _find_recurrence_work_item(recurrence, work_item_id)
|
|
source_review = _as_dict(item.get("source_correlation_review"))
|
|
target_id = (
|
|
str(
|
|
source_review.get("target_incident_id")
|
|
or work_item.get("matched_incident_id")
|
|
or ""
|
|
).strip()
|
|
or None
|
|
)
|
|
checks = _source_correlation_apply_checks(
|
|
item,
|
|
work_item,
|
|
source_review,
|
|
target_incident_id=target_id,
|
|
)
|
|
allowed = all(check["passed"] for check in checks)
|
|
raw_event_id = _provider_raw_event_id(item.get("latest_provider_event_id"))
|
|
provider = str(item.get("provider") or "external").strip().lower() or "external"
|
|
source_event_provider_event_id = (
|
|
f"{provider}:{_SOURCE_CORRELATION_APPLY_STAGE}:{raw_event_id[:96]}"
|
|
)
|
|
|
|
payload = {
|
|
"schema_version": _SOURCE_CORRELATION_APPLY_SCHEMA_VERSION,
|
|
"source": "channel_event_dossier.recurrence",
|
|
"project_id": recurrence.get("project_id"),
|
|
"work_item_id": work_item.get("work_item_id"),
|
|
"review_id": source_review.get("review_id"),
|
|
"target_incident_id": target_id,
|
|
"reviewer_id": str(reviewer_id or "operator_console")[:100],
|
|
"operator_note": str(operator_note or "").strip()[:500] or None,
|
|
"allowed": allowed,
|
|
"executed": allowed,
|
|
"apply_status": "ready_to_apply" if allowed else "blocked",
|
|
"safety_level": "source_review_append_only_apply",
|
|
"writes_incident_state": False,
|
|
"writes_source_event": allowed,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"creates_external_ticket": False,
|
|
"latest_provider_event_id": item.get("latest_provider_event_id"),
|
|
"source_event_provider_event_id": source_event_provider_event_id,
|
|
"source_event_stage": _SOURCE_CORRELATION_APPLY_STAGE,
|
|
"raw_event_id": raw_event_id,
|
|
"provider": provider,
|
|
"alertname": item.get("alertname"),
|
|
"severity": item.get("severity"),
|
|
"namespace": item.get("namespace"),
|
|
"target_resource": item.get("target_resource"),
|
|
"fingerprint": item.get("fingerprint"),
|
|
"checks": checks,
|
|
"current_state_summary": _recurrence_current_state_summary(item, work_item),
|
|
"plan": {
|
|
"step": "append_source_correlation_link_event",
|
|
"flywheel_node": "source_correlation_apply",
|
|
"agent_id": "awooop_source_correlation_reviewer",
|
|
"required_scope": "append_source_link",
|
|
"writes": (
|
|
["awooop_conversation_event", "timeline_events", "alert_operation_log"]
|
|
if allowed
|
|
else []
|
|
),
|
|
"target_action": "verify_source_link_in_status_chain",
|
|
"reason": (
|
|
"provider_native_evidence_link_applied"
|
|
if allowed
|
|
else "source_correlation_apply_preflight_failed"
|
|
),
|
|
"target": _recurrence_work_item_target(item),
|
|
},
|
|
"read_model_route": {
|
|
"agent_id": "awooop_source_correlation_reviewer",
|
|
"tool_name": "channel_event_dossier.recurrence",
|
|
"required_scope": "append_source_link",
|
|
"is_shadow": False,
|
|
"flywheel_node": "source_correlation_apply",
|
|
},
|
|
"next_step": (
|
|
"verify_source_link_in_status_chain"
|
|
if allowed
|
|
else "fix_preflight_checks"
|
|
),
|
|
}
|
|
if not allowed:
|
|
payload["executed"] = False
|
|
return payload
|
|
|
|
|
|
async def _record_source_correlation_apply_history(
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if not payload.get("allowed"):
|
|
return {"recorded": False, "reason": "preflight_failed"}
|
|
|
|
history: dict[str, Any] = {
|
|
"recorded": False,
|
|
"source_event_id": None,
|
|
"alert_operation_id": None,
|
|
"timeline_event_id": None,
|
|
}
|
|
incident_id = str(payload.get("target_incident_id") or "")
|
|
|
|
try:
|
|
from src.services.channel_hub import record_external_alert_event
|
|
|
|
event_id = await record_external_alert_event(
|
|
project_id=str(payload.get("project_id") or "awoooi"),
|
|
provider=str(payload.get("provider") or "external"),
|
|
event_id=str(payload.get("raw_event_id") or payload.get("work_item_id")),
|
|
stage=_SOURCE_CORRELATION_APPLY_STAGE,
|
|
title=str(payload.get("alertname") or "Source correlation link"),
|
|
severity=str(payload.get("severity") or "info"),
|
|
namespace=payload.get("namespace"),
|
|
target_resource=payload.get("target_resource"),
|
|
fingerprint=payload.get("fingerprint"),
|
|
incident_id=incident_id,
|
|
source_url=None,
|
|
labels={
|
|
"awooop_source_correlation": "applied",
|
|
"work_item_id": payload.get("work_item_id"),
|
|
},
|
|
annotations={
|
|
"review_id": payload.get("review_id") or "",
|
|
"original_provider_event_id": payload.get("latest_provider_event_id")
|
|
or "",
|
|
},
|
|
payload={
|
|
"schema_version": _SOURCE_CORRELATION_APPLY_SCHEMA_VERSION,
|
|
"work_item_id": payload.get("work_item_id"),
|
|
"review_id": payload.get("review_id"),
|
|
"target_incident_id": incident_id,
|
|
},
|
|
is_duplicate=False,
|
|
)
|
|
if event_id:
|
|
history["source_event_id"] = str(event_id)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"awooop_source_correlation_apply_source_event_failed",
|
|
work_item_id=payload.get("work_item_id"),
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
context = _source_correlation_apply_context({**payload, "history": history})
|
|
try:
|
|
from src.services.approval_db import get_timeline_service
|
|
|
|
event = await get_timeline_service().add_event(
|
|
event_type="human",
|
|
status="success" if history.get("source_event_id") else "warning",
|
|
title="AwoooP source correlation apply",
|
|
description=_source_correlation_apply_description(context),
|
|
actor=_SOURCE_CORRELATION_APPLY_ACTOR,
|
|
actor_role="source_correlation_apply",
|
|
incident_id=incident_id,
|
|
)
|
|
if event:
|
|
history["timeline_event_id"] = event.get("id")
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"awooop_source_correlation_apply_timeline_failed",
|
|
incident_id=incident_id,
|
|
work_item_id=payload.get("work_item_id"),
|
|
error=str(exc),
|
|
)
|
|
|
|
try:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
final_context = _source_correlation_apply_context({
|
|
**payload,
|
|
"apply_status": "applied" if history.get("source_event_id") else "partial",
|
|
"history": history,
|
|
})
|
|
record = await get_alert_operation_log_repository().append(
|
|
"USER_ACTION",
|
|
incident_id=incident_id,
|
|
actor=_SOURCE_CORRELATION_APPLY_ACTOR,
|
|
action_detail="source_correlation_apply:append_source_link",
|
|
success=bool(history.get("source_event_id")),
|
|
context=_json_safe(final_context),
|
|
)
|
|
if record is not None:
|
|
history["alert_operation_id"] = getattr(record, "id", None)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"awooop_source_correlation_apply_alert_operation_failed",
|
|
work_item_id=payload.get("work_item_id"),
|
|
error=str(exc),
|
|
)
|
|
|
|
history["recorded"] = bool(
|
|
history.get("source_event_id")
|
|
or history.get("alert_operation_id")
|
|
or history.get("timeline_event_id")
|
|
)
|
|
if not history["recorded"]:
|
|
history["reason"] = "history_sink_unavailable"
|
|
return history
|
|
|
|
|
|
def build_dossier_coverage(
|
|
rows: list[dict[str, Any]],
|
|
*,
|
|
project_id: str,
|
|
limit: int,
|
|
) -> dict[str, Any]:
|
|
"""Summarize recent inbound source envelopes for console coverage checks."""
|
|
events = [build_dossier_event(row) for row in rows]
|
|
provider_map: dict[str, dict[str, Any]] = {}
|
|
source_envelope_total = 0
|
|
with_source_refs_total = 0
|
|
sentry_ref_total = 0
|
|
signoz_ref_total = 0
|
|
alert_ref_total = 0
|
|
|
|
for row, event in zip(rows, events, strict=False):
|
|
envelope = _as_dict(row.get("source_envelope"))
|
|
if envelope:
|
|
source_envelope_total += 1
|
|
|
|
source_refs = _as_dict(event.get("source_refs"))
|
|
source_ref_count = int(event.get("source_ref_count") or 0)
|
|
if source_ref_count > 0:
|
|
with_source_refs_total += 1
|
|
|
|
provider = str(event.get("provider") or row.get("channel_type") or "unknown")
|
|
provider_item = provider_map.setdefault(
|
|
provider,
|
|
{
|
|
"provider": provider,
|
|
"total": 0,
|
|
"duplicate_total": 0,
|
|
"redacted_total": 0,
|
|
"source_ref_total": 0,
|
|
"missing_source_refs_total": 0,
|
|
"sentry_ref_total": 0,
|
|
"signoz_ref_total": 0,
|
|
"alert_ref_total": 0,
|
|
"latest_received_at": None,
|
|
},
|
|
)
|
|
provider_item["total"] += 1
|
|
provider_item["source_ref_total"] += source_ref_count
|
|
if event.get("is_duplicate"):
|
|
provider_item["duplicate_total"] += 1
|
|
if event.get("has_redacted_content"):
|
|
provider_item["redacted_total"] += 1
|
|
if source_ref_count <= 0:
|
|
provider_item["missing_source_refs_total"] += 1
|
|
|
|
event_sentry_refs = _ref_count(source_refs, "sentry_issue_ids")
|
|
event_signoz_refs = _ref_count(source_refs, "signoz_alerts")
|
|
event_alert_refs = _ref_count(source_refs, "alert_ids")
|
|
sentry_ref_total += event_sentry_refs
|
|
signoz_ref_total += event_signoz_refs
|
|
alert_ref_total += event_alert_refs
|
|
provider_item["sentry_ref_total"] += event_sentry_refs
|
|
provider_item["signoz_ref_total"] += event_signoz_refs
|
|
provider_item["alert_ref_total"] += event_alert_refs
|
|
provider_item["latest_received_at"] = provider_item[
|
|
"latest_received_at"
|
|
] or event.get("received_at")
|
|
|
|
duplicate_total = sum(1 for event in events if event.get("is_duplicate"))
|
|
redacted_total = sum(1 for event in events if event.get("has_redacted_content"))
|
|
source_ref_total = sum(int(event.get("source_ref_count") or 0) for event in events)
|
|
missing_source_refs_total = len(events) - with_source_refs_total
|
|
missing_source_envelope_total = len(events) - source_envelope_total
|
|
|
|
return {
|
|
"project_id": project_id,
|
|
"limit": limit,
|
|
"summary": {
|
|
"source_count": len(events),
|
|
"source_envelope_total": source_envelope_total,
|
|
"missing_source_envelope_total": missing_source_envelope_total,
|
|
"with_source_refs_total": with_source_refs_total,
|
|
"missing_source_refs_total": missing_source_refs_total,
|
|
"duplicate_total": duplicate_total,
|
|
"redacted_total": redacted_total,
|
|
"source_ref_total": source_ref_total,
|
|
"sentry_ref_total": sentry_ref_total,
|
|
"signoz_ref_total": signoz_ref_total,
|
|
"alert_ref_total": alert_ref_total,
|
|
"latest_received_at": events[0].get("received_at") if events else None,
|
|
},
|
|
"providers": sorted(
|
|
provider_map.values(),
|
|
key=lambda item: (
|
|
-int(item.get("total") or 0),
|
|
str(item.get("provider") or ""),
|
|
),
|
|
),
|
|
}
|
|
|
|
|
|
def build_dossier_event(row: dict[str, Any]) -> dict[str, Any]:
|
|
"""Normalize a DB row into the front-end event dossier shape."""
|
|
envelope = _as_dict(row.get("source_envelope"))
|
|
source_refs = _as_dict(envelope.get("source_refs"))
|
|
log_correlation = _as_dict(envelope.get("log_correlation"))
|
|
content_redacted = row.get("content_redacted")
|
|
content_preview = row.get("content_preview")
|
|
|
|
return {
|
|
"event_id": row.get("event_id"),
|
|
"project_id": row.get("project_id"),
|
|
"channel_type": row.get("channel_type"),
|
|
"provider": envelope.get("provider") or row.get("channel_type"),
|
|
"stage": envelope.get("stage") or "received",
|
|
"provider_event_id": row.get("provider_event_id"),
|
|
"content_preview": content_preview,
|
|
"content_redacted": content_redacted,
|
|
"has_redacted_content": bool(content_redacted),
|
|
"redaction_version": row.get("redaction_version"),
|
|
"source_url": envelope.get("source_url"),
|
|
"content_sha256": envelope.get("content_sha256") or row.get("content_hash"),
|
|
"content_length": envelope.get("content_length"),
|
|
"source_refs": source_refs,
|
|
"source_ref_count": _compact_ref_count(source_refs),
|
|
"log_correlation": log_correlation,
|
|
"alertname": log_correlation.get("alertname"),
|
|
"severity": log_correlation.get("severity"),
|
|
"namespace": log_correlation.get("namespace"),
|
|
"target_resource": log_correlation.get("target_resource"),
|
|
"fingerprint": log_correlation.get("fingerprint"),
|
|
"is_duplicate": row.get("is_duplicate"),
|
|
"provider_ts": row.get("provider_ts"),
|
|
"received_at": row.get("received_at"),
|
|
}
|
|
|
|
|
|
def _collect_incident_ids_from_rows(rows: list[dict[str, Any]]) -> list[str]:
|
|
incident_ids: list[str] = []
|
|
for row in rows:
|
|
event = build_dossier_event(row)
|
|
for incident_id in _event_incident_ids(event):
|
|
_append_unique(incident_ids, incident_id)
|
|
return incident_ids
|
|
|
|
|
|
async def _fetch_auto_repair_summaries_by_incident(
|
|
db: Any,
|
|
incident_ids: list[str],
|
|
) -> dict[str, dict[str, Any]]:
|
|
"""Fetch latest auto-repair and verifier evidence for recurrence groups."""
|
|
visible_incident_ids = incident_ids[:_MAX_REPAIR_INCIDENTS]
|
|
if not visible_incident_ids:
|
|
return {}
|
|
|
|
placeholders: list[str] = []
|
|
params: dict[str, Any] = {}
|
|
for index, incident_id in enumerate(visible_incident_ids):
|
|
key = f"incident_id_{index}"
|
|
placeholders.append(f":{key}")
|
|
params[key] = incident_id
|
|
|
|
result = await db.execute(
|
|
text(
|
|
f"""
|
|
WITH ranked AS (
|
|
SELECT
|
|
are.id AS latest_auto_repair_id,
|
|
are.incident_id,
|
|
are.playbook_id AS latest_playbook_id,
|
|
are.playbook_name AS latest_playbook_name,
|
|
are.success AS latest_success,
|
|
left(coalesce(are.error_message, ''), 240) AS latest_error_message_preview,
|
|
are.triggered_by AS latest_triggered_by,
|
|
are.risk_level AS latest_risk_level,
|
|
are.execution_time_ms AS latest_execution_time_ms,
|
|
are.created_at AS latest_auto_repair_at,
|
|
latest_evidence.verification_result AS latest_verification_result,
|
|
latest_evidence.collected_at AS latest_verification_at,
|
|
COUNT(*) OVER (PARTITION BY are.incident_id) AS auto_repair_total,
|
|
COUNT(*) FILTER (WHERE are.success IS TRUE)
|
|
OVER (PARTITION BY are.incident_id) AS success_total,
|
|
COUNT(*) FILTER (WHERE are.success IS FALSE)
|
|
OVER (PARTITION BY are.incident_id) AS failed_total,
|
|
ROW_NUMBER() OVER (
|
|
PARTITION BY are.incident_id
|
|
ORDER BY are.created_at DESC
|
|
) AS rn
|
|
FROM auto_repair_executions are
|
|
LEFT JOIN LATERAL (
|
|
SELECT
|
|
ev.verification_result,
|
|
ev.collected_at
|
|
FROM incident_evidence ev
|
|
WHERE ev.incident_id = are.incident_id
|
|
AND ev.verification_result IS NOT NULL
|
|
ORDER BY ev.collected_at DESC
|
|
LIMIT 1
|
|
) latest_evidence ON TRUE
|
|
WHERE are.incident_id IN ({", ".join(placeholders)})
|
|
)
|
|
SELECT
|
|
latest_auto_repair_id,
|
|
incident_id,
|
|
latest_playbook_id,
|
|
latest_playbook_name,
|
|
latest_success,
|
|
latest_error_message_preview,
|
|
latest_triggered_by,
|
|
latest_risk_level,
|
|
latest_execution_time_ms,
|
|
latest_auto_repair_at,
|
|
latest_verification_result,
|
|
latest_verification_at,
|
|
auto_repair_total,
|
|
success_total,
|
|
failed_total
|
|
FROM ranked
|
|
WHERE rn = 1
|
|
"""
|
|
),
|
|
params,
|
|
)
|
|
|
|
summaries: dict[str, dict[str, Any]] = {}
|
|
for row in result.mappings().all():
|
|
item = dict(row)
|
|
incident_id = str(item.get("incident_id") or "")
|
|
if not incident_id:
|
|
continue
|
|
item["schema_version"] = "awooop_recurrence_repair_summary_v1"
|
|
summaries[incident_id] = item
|
|
return summaries
|
|
|
|
|
|
async def _fetch_source_review_decisions_by_work_item(
|
|
db: Any,
|
|
*,
|
|
project_id: str,
|
|
limit: int,
|
|
) -> dict[str, dict[str, Any]]:
|
|
"""Fetch latest source-correlation review decisions from event-sourced audit."""
|
|
|
|
result = await db.execute(
|
|
text("""
|
|
WITH ranked AS (
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
actor,
|
|
action_detail,
|
|
success,
|
|
context,
|
|
created_at,
|
|
ROW_NUMBER() OVER (
|
|
PARTITION BY context->>'work_item_id'
|
|
ORDER BY created_at DESC
|
|
) AS rn
|
|
FROM alert_operation_log
|
|
WHERE actor = :actor
|
|
AND context->>'schema_version' = :schema_version
|
|
AND COALESCE(context->>'project_id', :project_id) = :project_id
|
|
)
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
actor,
|
|
action_detail,
|
|
success,
|
|
context,
|
|
created_at
|
|
FROM ranked
|
|
WHERE rn = 1
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
"""),
|
|
{
|
|
"actor": _SOURCE_CORRELATION_REVIEW_ACTOR,
|
|
"schema_version": _SOURCE_CORRELATION_DECISION_SCHEMA_VERSION,
|
|
"project_id": project_id,
|
|
"limit": max(1, min(limit, _MAX_RECURRENCE_EVENTS)),
|
|
},
|
|
)
|
|
|
|
decisions: dict[str, dict[str, Any]] = {}
|
|
for row in result.mappings().all():
|
|
item = _normalize_source_review_decision(dict(row))
|
|
if not item:
|
|
continue
|
|
decisions[str(item["work_item_id"])] = item
|
|
return decisions
|
|
|
|
|
|
async def _fetch_source_applies_by_work_item(
|
|
db: Any,
|
|
*,
|
|
project_id: str,
|
|
limit: int,
|
|
) -> dict[str, dict[str, Any]]:
|
|
"""Fetch latest source-correlation apply records from event-sourced audit."""
|
|
|
|
result = await db.execute(
|
|
text("""
|
|
WITH ranked AS (
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
actor,
|
|
action_detail,
|
|
success,
|
|
context,
|
|
created_at,
|
|
ROW_NUMBER() OVER (
|
|
PARTITION BY context->>'work_item_id'
|
|
ORDER BY created_at DESC
|
|
) AS rn
|
|
FROM alert_operation_log
|
|
WHERE actor = :actor
|
|
AND context->>'schema_version' = :schema_version
|
|
AND COALESCE(context->>'project_id', :project_id) = :project_id
|
|
)
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
actor,
|
|
action_detail,
|
|
success,
|
|
context,
|
|
created_at
|
|
FROM ranked
|
|
WHERE rn = 1
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
"""),
|
|
{
|
|
"actor": _SOURCE_CORRELATION_APPLY_ACTOR,
|
|
"schema_version": _SOURCE_CORRELATION_APPLY_SCHEMA_VERSION,
|
|
"project_id": project_id,
|
|
"limit": max(1, min(limit, _MAX_RECURRENCE_EVENTS)),
|
|
},
|
|
)
|
|
|
|
applies: dict[str, dict[str, Any]] = {}
|
|
for row in result.mappings().all():
|
|
item = _normalize_source_apply(dict(row))
|
|
if not item:
|
|
continue
|
|
applies[str(item["work_item_id"])] = item
|
|
return applies
|
|
|
|
|
|
async def fetch_channel_event_dossier(
|
|
*,
|
|
project_id: str | None,
|
|
run_id: UUID | None,
|
|
provider_event_id: str | None,
|
|
limit: int,
|
|
) -> dict[str, Any]:
|
|
"""Fetch redacted source envelopes for a run or provider event id."""
|
|
if run_id is None and not provider_event_id:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
detail="run_id or provider_event_id is required",
|
|
)
|
|
|
|
effective_project_id = project_id or "awoooi"
|
|
safe_limit = max(1, min(limit, _MAX_DOSSIER_EVENTS))
|
|
where_clauses = ["project_id = :project_id"]
|
|
params: dict[str, Any] = {
|
|
"project_id": effective_project_id,
|
|
"limit": safe_limit,
|
|
}
|
|
if run_id is not None:
|
|
where_clauses.append("run_id = CAST(:run_id AS uuid)")
|
|
params["run_id"] = str(run_id)
|
|
if provider_event_id:
|
|
where_clauses.append("provider_event_id = :provider_event_id")
|
|
params["provider_event_id"] = provider_event_id
|
|
|
|
async with get_db_context(effective_project_id) as db:
|
|
result = await db.execute(
|
|
text(
|
|
f"""
|
|
SELECT
|
|
event_id,
|
|
project_id,
|
|
channel_type,
|
|
provider_event_id,
|
|
content_hash,
|
|
content_preview,
|
|
content_redacted,
|
|
redaction_version,
|
|
source_envelope,
|
|
is_duplicate,
|
|
provider_ts,
|
|
received_at
|
|
FROM awooop_conversation_event
|
|
WHERE {" AND ".join(where_clauses)}
|
|
ORDER BY received_at ASC
|
|
LIMIT :limit
|
|
"""
|
|
),
|
|
params,
|
|
)
|
|
rows = [dict(row) for row in result.mappings().all()]
|
|
|
|
events = [build_dossier_event(row) for row in rows]
|
|
duplicate_total = sum(1 for event in events if event.get("is_duplicate"))
|
|
redacted_total = sum(1 for event in events if event.get("has_redacted_content"))
|
|
|
|
return {
|
|
"events": events,
|
|
"total": len(events),
|
|
"limit": safe_limit,
|
|
"summary": {
|
|
"source_count": len(events),
|
|
"duplicate_total": duplicate_total,
|
|
"redacted_total": redacted_total,
|
|
"source_ref_total": sum(
|
|
int(event.get("source_ref_count") or 0) for event in events
|
|
),
|
|
},
|
|
}
|
|
|
|
|
|
async def fetch_channel_event_dossier_coverage(
|
|
*,
|
|
project_id: str | None,
|
|
provider: str | None,
|
|
limit: int,
|
|
) -> dict[str, Any]:
|
|
"""Fetch a read-only coverage summary for recent inbound channel events."""
|
|
effective_project_id = project_id or "awoooi"
|
|
safe_limit = max(1, min(limit, _MAX_COVERAGE_EVENTS))
|
|
where_clauses = ["project_id = :project_id"]
|
|
params: dict[str, Any] = {
|
|
"project_id": effective_project_id,
|
|
"limit": safe_limit,
|
|
}
|
|
if provider:
|
|
where_clauses.append(
|
|
"COALESCE(NULLIF(source_envelope->>'provider', ''), "
|
|
"split_part(provider_event_id, ':', 1), channel_type) = :provider"
|
|
)
|
|
params["provider"] = provider
|
|
|
|
async with get_db_context(effective_project_id) as db:
|
|
result = await db.execute(
|
|
text(
|
|
f"""
|
|
SELECT
|
|
event_id,
|
|
project_id,
|
|
channel_type,
|
|
provider_event_id,
|
|
content_hash,
|
|
content_preview,
|
|
content_redacted,
|
|
redaction_version,
|
|
source_envelope,
|
|
is_duplicate,
|
|
provider_ts,
|
|
received_at
|
|
FROM awooop_conversation_event
|
|
WHERE {" AND ".join(where_clauses)}
|
|
ORDER BY received_at DESC
|
|
LIMIT :limit
|
|
"""
|
|
),
|
|
params,
|
|
)
|
|
rows = [dict(row) for row in result.mappings().all()]
|
|
|
|
return build_dossier_coverage(
|
|
rows,
|
|
project_id=effective_project_id,
|
|
limit=safe_limit,
|
|
)
|
|
|
|
|
|
async def fetch_channel_event_dossier_recurrence(
|
|
*,
|
|
project_id: str | None,
|
|
provider: str | None,
|
|
limit: int,
|
|
) -> dict[str, Any]:
|
|
"""Fetch recurrence groups and linked run state for recent source events."""
|
|
effective_project_id = project_id or "awoooi"
|
|
safe_limit = max(1, min(limit, _MAX_RECURRENCE_EVENTS))
|
|
where_clauses = ["e.project_id = :project_id"]
|
|
params: dict[str, Any] = {
|
|
"project_id": effective_project_id,
|
|
"limit": safe_limit,
|
|
}
|
|
if provider:
|
|
where_clauses.append(
|
|
"COALESCE(NULLIF(e.source_envelope->>'provider', ''), "
|
|
"split_part(e.provider_event_id, ':', 1), e.channel_type) = :provider"
|
|
)
|
|
params["provider"] = provider
|
|
|
|
async with get_db_context(effective_project_id) as db:
|
|
result = await db.execute(
|
|
text(
|
|
f"""
|
|
SELECT
|
|
e.event_id,
|
|
e.project_id,
|
|
e.channel_type,
|
|
e.provider_event_id,
|
|
e.content_hash,
|
|
e.content_preview,
|
|
e.content_redacted,
|
|
e.redaction_version,
|
|
e.source_envelope,
|
|
e.is_duplicate,
|
|
e.provider_ts,
|
|
e.received_at,
|
|
e.run_id,
|
|
r.state AS run_state,
|
|
r.agent_id AS run_agent_id
|
|
FROM awooop_conversation_event e
|
|
LEFT JOIN awooop_run_state r
|
|
ON r.project_id = e.project_id
|
|
AND r.run_id = e.run_id
|
|
WHERE {" AND ".join(where_clauses)}
|
|
ORDER BY e.received_at DESC
|
|
LIMIT :limit
|
|
"""
|
|
),
|
|
params,
|
|
)
|
|
rows = [dict(row) for row in result.mappings().all()]
|
|
repair_summaries = await _fetch_auto_repair_summaries_by_incident(
|
|
db,
|
|
_collect_incident_ids_from_rows(rows),
|
|
)
|
|
source_review_decisions = await _fetch_source_review_decisions_by_work_item(
|
|
db,
|
|
project_id=effective_project_id,
|
|
limit=safe_limit,
|
|
)
|
|
source_applies = await _fetch_source_applies_by_work_item(
|
|
db,
|
|
project_id=effective_project_id,
|
|
limit=safe_limit,
|
|
)
|
|
|
|
return build_dossier_recurrence(
|
|
rows,
|
|
project_id=effective_project_id,
|
|
limit=safe_limit,
|
|
repair_summaries_by_incident=repair_summaries,
|
|
source_review_decisions_by_work_item=source_review_decisions,
|
|
source_applies_by_work_item=source_applies,
|
|
)
|
|
|
|
|
|
async def fetch_recurrence_work_item_preview(
|
|
*,
|
|
project_id: str | None,
|
|
work_item_id: str,
|
|
mode: RecurrenceWorkItemMode = "auto",
|
|
provider: str | None = None,
|
|
limit: int = _MAX_RECURRENCE_EVENTS,
|
|
) -> dict[str, Any]:
|
|
"""Fetch a read-only preview for a recurrence work item."""
|
|
|
|
recurrence = await fetch_channel_event_dossier_recurrence(
|
|
project_id=project_id,
|
|
provider=provider,
|
|
limit=limit,
|
|
)
|
|
return build_recurrence_work_item_preview(
|
|
recurrence,
|
|
work_item_id=work_item_id,
|
|
mode=mode,
|
|
)
|
|
|
|
|
|
async def fetch_recurrence_work_item_dry_run(
|
|
*,
|
|
project_id: str | None,
|
|
work_item_id: str,
|
|
mode: RecurrenceWorkItemMode = "auto",
|
|
provider: str | None = None,
|
|
limit: int = _MAX_RECURRENCE_EVENTS,
|
|
) -> dict[str, Any]:
|
|
"""Fetch and record a safe read-only dry-run for a recurrence work item."""
|
|
|
|
recurrence = await fetch_channel_event_dossier_recurrence(
|
|
project_id=project_id,
|
|
provider=provider,
|
|
limit=limit,
|
|
)
|
|
payload = build_recurrence_work_item_dry_run(
|
|
recurrence,
|
|
work_item_id=work_item_id,
|
|
mode=mode,
|
|
)
|
|
payload["history"] = await _record_recurrence_work_item_dry_run_history(payload)
|
|
return payload
|
|
|
|
|
|
async def fetch_recurrence_work_item_handoff(
|
|
*,
|
|
project_id: str | None,
|
|
work_item_id: str,
|
|
mode: RecurrenceWorkItemMode = "auto",
|
|
handoff_kind: RecurrenceWorkItemHandoffKind = "ticket_proposal",
|
|
provider: str | None = None,
|
|
limit: int = _MAX_RECURRENCE_EVENTS,
|
|
) -> dict[str, Any]:
|
|
"""Fetch and record a safe handoff proposal for a recurrence work item."""
|
|
|
|
recurrence = await fetch_channel_event_dossier_recurrence(
|
|
project_id=project_id,
|
|
provider=provider,
|
|
limit=limit,
|
|
)
|
|
payload = build_recurrence_work_item_handoff(
|
|
recurrence,
|
|
work_item_id=work_item_id,
|
|
mode=mode,
|
|
handoff_kind=handoff_kind,
|
|
)
|
|
payload["history"] = await _record_recurrence_work_item_handoff_history(payload)
|
|
if payload["history"].get("recorded"):
|
|
payload["handoff_status"] = "recorded"
|
|
elif payload.get("allowed"):
|
|
payload["handoff_status"] = "record_failed"
|
|
return payload
|
|
|
|
|
|
async def fetch_source_correlation_review_decision(
|
|
*,
|
|
project_id: str | None,
|
|
work_item_id: str,
|
|
decision: SourceCorrelationReviewDecision,
|
|
target_incident_id: str | None = None,
|
|
reviewer_id: str = "operator_console",
|
|
operator_note: str | None = None,
|
|
provider: str | None = None,
|
|
limit: int = _MAX_RECURRENCE_EVENTS,
|
|
) -> dict[str, Any]:
|
|
"""Fetch recurrence state and record an explicit source review decision."""
|
|
|
|
recurrence = await fetch_channel_event_dossier_recurrence(
|
|
project_id=project_id,
|
|
provider=provider,
|
|
limit=limit,
|
|
)
|
|
payload = build_source_correlation_review_decision(
|
|
recurrence,
|
|
work_item_id=work_item_id,
|
|
decision=decision,
|
|
target_incident_id=target_incident_id,
|
|
reviewer_id=reviewer_id,
|
|
operator_note=operator_note,
|
|
)
|
|
payload["history"] = await _record_source_correlation_review_decision_history(
|
|
payload
|
|
)
|
|
if payload["history"].get("recorded"):
|
|
payload["review_record_status"] = "recorded"
|
|
elif payload.get("allowed"):
|
|
payload["review_record_status"] = "record_failed"
|
|
else:
|
|
payload["review_record_status"] = "blocked"
|
|
return payload
|
|
|
|
|
|
async def fetch_source_correlation_apply(
|
|
*,
|
|
project_id: str | None,
|
|
work_item_id: str,
|
|
reviewer_id: str = "operator_console",
|
|
operator_note: str | None = None,
|
|
provider: str | None = None,
|
|
limit: int = _MAX_RECURRENCE_EVENTS,
|
|
) -> dict[str, Any]:
|
|
"""Fetch recurrence state and append an accepted source review link event."""
|
|
|
|
recurrence = await fetch_channel_event_dossier_recurrence(
|
|
project_id=project_id,
|
|
provider=provider,
|
|
limit=limit,
|
|
)
|
|
payload = build_source_correlation_apply(
|
|
recurrence,
|
|
work_item_id=work_item_id,
|
|
reviewer_id=reviewer_id,
|
|
operator_note=operator_note,
|
|
)
|
|
payload["history"] = await _record_source_correlation_apply_history(payload)
|
|
if payload["history"].get("source_event_id"):
|
|
payload["apply_status"] = "applied"
|
|
elif payload["history"].get("recorded") and payload.get("allowed"):
|
|
payload["apply_status"] = "partial"
|
|
elif payload.get("allowed"):
|
|
payload["apply_status"] = "record_failed"
|
|
else:
|
|
payload["apply_status"] = "blocked"
|
|
return payload
|