335 lines
12 KiB
Python
335 lines
12 KiB
Python
"""Cross-run source health, retry, fallback, and alert decisions for PChome sales."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, Iterable, List, Mapping, Optional
|
|
|
|
|
|
SOURCE_PRIORITY = (
|
|
"google_drive",
|
|
"authorized_https",
|
|
"authorized_imap",
|
|
"controlled_local_drop",
|
|
)
|
|
|
|
_EMPTY_STATUSES = {
|
|
"no_candidate",
|
|
"no_pending_file",
|
|
"upstream_missing",
|
|
"upstream_stale",
|
|
}
|
|
_FAILED_STATUSES = {"failed", "not_ready", "probe_failed"}
|
|
|
|
|
|
def _provider_category(
|
|
source: Mapping[str, object],
|
|
attempt: Optional[Mapping[str, object]],
|
|
) -> str:
|
|
if not bool(source.get("enabled")):
|
|
return "disabled"
|
|
if not attempt:
|
|
return "not_probed"
|
|
|
|
status = str(attempt.get("status") or "unknown")
|
|
if int(attempt.get("imported_count") or 0) > 0:
|
|
return "imported"
|
|
if int(attempt.get("duplicate_count") or 0) > 0 or status == "duplicate":
|
|
return "duplicate"
|
|
if int(attempt.get("candidate_count") or 0) > 0:
|
|
return "candidate_found"
|
|
if attempt.get("error_kind") or status in _FAILED_STATUSES:
|
|
return "failed"
|
|
if status in _EMPTY_STATUSES:
|
|
return "empty"
|
|
if not bool(source.get("ready")):
|
|
return "not_ready"
|
|
return "ready"
|
|
|
|
|
|
def _history_attempts(
|
|
receipts: Iterable[Mapping[str, object]],
|
|
source_id: str,
|
|
) -> List[Dict[str, object]]:
|
|
attempts: List[Dict[str, object]] = []
|
|
for receipt in receipts:
|
|
stages = receipt.get("stages") or {}
|
|
if not isinstance(stages, Mapping):
|
|
continue
|
|
providers = stages.get("providers") or []
|
|
if not isinstance(providers, list):
|
|
continue
|
|
for provider in providers:
|
|
if isinstance(provider, Mapping) and provider.get("source") == source_id:
|
|
attempts.append({
|
|
**dict(provider),
|
|
"receipt_status": receipt.get("status"),
|
|
"attempted_at": receipt.get("completed_at") or receipt.get("started_at"),
|
|
})
|
|
break
|
|
return attempts
|
|
|
|
|
|
def _consecutive_count(categories: Iterable[str], target: str) -> int:
|
|
count = 0
|
|
for category in categories:
|
|
if category != target:
|
|
break
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def _parse_datetime(value: object) -> Optional[datetime]:
|
|
if not value:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return parsed
|
|
|
|
|
|
def _lag_bucket(lag_days: object) -> str:
|
|
if lag_days is None:
|
|
return "missing"
|
|
try:
|
|
lag = int(lag_days)
|
|
except (TypeError, ValueError):
|
|
return "missing"
|
|
if lag <= 1:
|
|
return "fresh"
|
|
if lag <= 3:
|
|
return "warning"
|
|
if lag <= 7:
|
|
return "stale"
|
|
return "critical"
|
|
|
|
|
|
def _previous_alert(receipts: Iterable[Mapping[str, object]]) -> Dict[str, object]:
|
|
for receipt in receipts:
|
|
stages = receipt.get("stages") or {}
|
|
if not isinstance(stages, Mapping):
|
|
continue
|
|
reconciliation = stages.get("source_reconciliation") or {}
|
|
if not isinstance(reconciliation, Mapping):
|
|
continue
|
|
alert = reconciliation.get("notification_policy") or {}
|
|
if isinstance(alert, Mapping) and alert.get("fingerprint"):
|
|
return {
|
|
**dict(alert),
|
|
"completed_at": receipt.get("completed_at") or receipt.get("started_at"),
|
|
}
|
|
return {}
|
|
|
|
|
|
def build_source_reconciliation(
|
|
*,
|
|
sources: Iterable[Mapping[str, object]],
|
|
provider_attempts: Iterable[Mapping[str, object]],
|
|
prior_receipts: Iterable[Mapping[str, object]],
|
|
freshness: Mapping[str, object],
|
|
now: datetime,
|
|
retry_interval_minutes: int = 30,
|
|
max_failure_backoff_minutes: int = 240,
|
|
alert_cooldown_minutes: int = 360,
|
|
) -> Dict[str, object]:
|
|
"""Build a public-safe decision without exposing provider configuration values."""
|
|
source_list = [dict(source) for source in sources]
|
|
attempt_by_source = {
|
|
str(item.get("source")): dict(item)
|
|
for item in provider_attempts
|
|
if item.get("source")
|
|
}
|
|
receipt_list = list(prior_receipts)
|
|
source_states: List[Dict[str, object]] = []
|
|
|
|
for source in source_list:
|
|
source_id = str(source.get("id") or "unknown")
|
|
attempt = attempt_by_source.get(source_id)
|
|
category = _provider_category(source, attempt)
|
|
history = _history_attempts(receipt_list, source_id)
|
|
historical_categories = [
|
|
_provider_category(source, historical_attempt)
|
|
for historical_attempt in history
|
|
]
|
|
current_and_history = [category, *historical_categories]
|
|
consecutive_empty = _consecutive_count(current_and_history, "empty")
|
|
consecutive_failures = _consecutive_count(current_and_history, "failed")
|
|
|
|
retry_minutes = None
|
|
next_retry_at = None
|
|
if bool(source.get("enabled")) and category in {"duplicate", "empty", "failed", "not_ready"}:
|
|
retry_minutes = max(1, retry_interval_minutes)
|
|
if category in {"failed", "not_ready"}:
|
|
failure_run = max(consecutive_failures, 1)
|
|
retry_minutes = min(
|
|
retry_minutes * (2 ** min(failure_run - 1, 3)),
|
|
max_failure_backoff_minutes,
|
|
)
|
|
next_retry_at = (now + timedelta(minutes=retry_minutes)).isoformat()
|
|
|
|
action = {
|
|
"disabled": "activate_when_authorized_configuration_is_ready",
|
|
"not_probed": "probe_on_next_scheduler_run",
|
|
"not_ready": "reconcile_provider_then_retry",
|
|
"failed": "retry_with_bounded_backoff_and_probe_fallback",
|
|
"duplicate": "retry_on_schedule_and_probe_fallback",
|
|
"empty": "retry_on_schedule_and_probe_fallback",
|
|
"candidate_found": "validate_and_import_candidate",
|
|
"imported": "verify_freshness_and_monitor",
|
|
"ready": "monitor_on_schedule",
|
|
}[category]
|
|
source_states.append({
|
|
"id": source_id,
|
|
"enabled": bool(source.get("enabled")),
|
|
"ready": bool(source.get("ready")),
|
|
"configuration_status": source.get("status") or "unknown",
|
|
"runtime_state": category,
|
|
"error_kind": attempt.get("error_kind") if attempt else None,
|
|
"candidate_count": int((attempt or {}).get("candidate_count") or 0),
|
|
"imported_count": int((attempt or {}).get("imported_count") or 0),
|
|
"duplicate_count": int((attempt or {}).get("duplicate_count") or 0),
|
|
"last_attempt_at": (
|
|
now.isoformat()
|
|
if attempt and category != "disabled"
|
|
else history[0].get("attempted_at")
|
|
if history
|
|
else None
|
|
),
|
|
"last_success_at": (
|
|
now.isoformat()
|
|
if category == "imported"
|
|
else next(
|
|
(
|
|
historical_attempt.get("attempted_at")
|
|
for historical_attempt in history
|
|
if _provider_category(source, historical_attempt) == "imported"
|
|
),
|
|
None,
|
|
)
|
|
),
|
|
"consecutive_empty_runs": consecutive_empty,
|
|
"consecutive_failure_runs": consecutive_failures,
|
|
"retry_minutes": retry_minutes,
|
|
"next_retry_at": next_retry_at,
|
|
"action": action,
|
|
})
|
|
|
|
enabled_states = [item for item in source_states if item["enabled"]]
|
|
ready_states = [item for item in source_states if item["ready"]]
|
|
due_times = [
|
|
str(item["next_retry_at"])
|
|
for item in source_states
|
|
if item.get("next_retry_at")
|
|
]
|
|
decision_ready = bool(freshness.get("decision_ready"))
|
|
any_candidate = any(
|
|
item["runtime_state"] in {"candidate_found", "imported"}
|
|
for item in source_states
|
|
)
|
|
|
|
if decision_ready:
|
|
status = "closed"
|
|
automation_decision = "monitor_scheduled_sources"
|
|
safe_next_action = "none"
|
|
elif any_candidate:
|
|
status = "partial"
|
|
automation_decision = "verify_candidate_import_and_freshness"
|
|
safe_next_action = "post_import_freshness_verification"
|
|
elif ready_states:
|
|
status = "degraded"
|
|
automation_decision = "retry_ready_sources_and_probe_enabled_fallbacks"
|
|
safe_next_action = "scheduled_retry_and_enabled_fallback_probe"
|
|
elif enabled_states:
|
|
status = "blocked"
|
|
automation_decision = "repair_enabled_sources_with_bounded_retry"
|
|
safe_next_action = "reconcile_enabled_source_health_then_retry"
|
|
else:
|
|
status = "blocked"
|
|
automation_decision = "reconcile_authorized_source_configuration"
|
|
safe_next_action = "reconcile_authorized_source_configuration"
|
|
|
|
effective_lag_days = freshness.get("sla_lag_days")
|
|
if effective_lag_days is None:
|
|
effective_lag_days = freshness.get("data_lag_days")
|
|
lag_bucket = _lag_bucket(effective_lag_days)
|
|
severity = (
|
|
"info"
|
|
if decision_ready
|
|
else "critical"
|
|
if lag_bucket in {"critical", "missing"} or not ready_states
|
|
else "warning"
|
|
if lag_bucket in {"warning", "stale"}
|
|
else "info"
|
|
)
|
|
fingerprint_payload = {
|
|
"lag_bucket": lag_bucket,
|
|
"decision_ready": decision_ready,
|
|
"expected_latest_sales_date": freshness.get("expected_latest_sales_date"),
|
|
"freshness_contract": freshness.get("contract_version"),
|
|
"source_states": [
|
|
[item["id"], item["runtime_state"], item.get("error_kind")]
|
|
for item in source_states
|
|
],
|
|
}
|
|
fingerprint = hashlib.sha256(
|
|
json.dumps(fingerprint_payload, sort_keys=True, ensure_ascii=True).encode("utf-8")
|
|
).hexdigest()[:20]
|
|
previous_alert = _previous_alert(receipt_list)
|
|
previous_at = _parse_datetime(previous_alert.get("completed_at"))
|
|
elapsed_minutes = None
|
|
if previous_at:
|
|
comparable_now = now
|
|
if comparable_now.tzinfo and not previous_at.tzinfo:
|
|
previous_at = previous_at.replace(tzinfo=comparable_now.tzinfo)
|
|
elif previous_at.tzinfo and not comparable_now.tzinfo:
|
|
comparable_now = comparable_now.replace(tzinfo=previous_at.tzinfo)
|
|
elapsed_minutes = max(0, int((comparable_now - previous_at).total_seconds() // 60))
|
|
state_changed = previous_alert.get("fingerprint") != fingerprint
|
|
send_notification = severity in {"warning", "critical"} and (
|
|
state_changed
|
|
or elapsed_minutes is None
|
|
or elapsed_minutes >= max(1, alert_cooldown_minutes)
|
|
)
|
|
|
|
notification_policy = {
|
|
"event": "pchome_sales_source_freshness",
|
|
"severity": severity,
|
|
"fingerprint": fingerprint,
|
|
"send": send_notification,
|
|
"reason": (
|
|
"state_changed"
|
|
if state_changed
|
|
else "cooldown_elapsed"
|
|
if send_notification
|
|
else "duplicate_within_cooldown"
|
|
),
|
|
"cooldown_minutes": max(1, alert_cooldown_minutes),
|
|
}
|
|
|
|
return {
|
|
"contract": "pchome_sales_source_reconciliation_v1",
|
|
"status": status,
|
|
"automation_decision": automation_decision,
|
|
"manual_review_required": False,
|
|
"freshness": dict(freshness),
|
|
"source_priority": [
|
|
source_id
|
|
for source_id in SOURCE_PRIORITY
|
|
if any(item["id"] == source_id for item in source_states)
|
|
],
|
|
"fallback_chain": [item["id"] for item in ready_states],
|
|
"single_source_dependency": len(ready_states) == 1,
|
|
"coverage": {
|
|
"ready": len(ready_states),
|
|
"enabled": len(enabled_states),
|
|
"total": len(source_states),
|
|
},
|
|
"sources": source_states,
|
|
"next_retry_at": min(due_times) if due_times else None,
|
|
"safe_next_action": safe_next_action,
|
|
"notification_policy": notification_policy,
|
|
}
|