"""Cross-run source health, retry, fallback, and alert decisions for PChome sales.""" from __future__ import annotations import hashlib import json from datetime import datetime, timedelta from typing import Dict, Iterable, List, Mapping, Optional SOURCE_PRIORITY = ( "google_drive", "authorized_https", "authorized_imap", "controlled_local_drop", ) _EMPTY_STATUSES = { "no_candidate", "no_pending_file", "upstream_missing", "upstream_stale", } _FAILED_STATUSES = {"failed", "not_ready", "probe_failed"} def _provider_category( source: Mapping[str, object], attempt: Optional[Mapping[str, object]], ) -> str: if not bool(source.get("enabled")): return "disabled" if not attempt: return "not_probed" status = str(attempt.get("status") or "unknown") if int(attempt.get("imported_count") or 0) > 0: return "imported" if int(attempt.get("duplicate_count") or 0) > 0 or status == "duplicate": return "duplicate" if int(attempt.get("candidate_count") or 0) > 0: return "candidate_found" if attempt.get("error_kind") or status in _FAILED_STATUSES: return "failed" if status in _EMPTY_STATUSES: return "empty" if not bool(source.get("ready")): return "not_ready" return "ready" def _history_attempts( receipts: Iterable[Mapping[str, object]], source_id: str, ) -> List[Dict[str, object]]: attempts: List[Dict[str, object]] = [] for receipt in receipts: stages = receipt.get("stages") or {} if not isinstance(stages, Mapping): continue providers = stages.get("providers") or [] if not isinstance(providers, list): continue for provider in providers: if isinstance(provider, Mapping) and provider.get("source") == source_id: attempts.append({ **dict(provider), "receipt_status": receipt.get("status"), "attempted_at": receipt.get("completed_at") or receipt.get("started_at"), }) break return attempts def _consecutive_count(categories: Iterable[str], target: str) -> int: count = 0 for category in categories: if category != target: break count += 1 return count def _parse_datetime(value: object) -> Optional[datetime]: if not value: return None try: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) except (TypeError, ValueError): return None return parsed def _lag_bucket(lag_days: object) -> str: if lag_days is None: return "missing" try: lag = int(lag_days) except (TypeError, ValueError): return "missing" if lag <= 1: return "fresh" if lag <= 3: return "warning" if lag <= 7: return "stale" return "critical" def _previous_alert(receipts: Iterable[Mapping[str, object]]) -> Dict[str, object]: for receipt in receipts: stages = receipt.get("stages") or {} if not isinstance(stages, Mapping): continue reconciliation = stages.get("source_reconciliation") or {} if not isinstance(reconciliation, Mapping): continue alert = reconciliation.get("notification_policy") or {} if isinstance(alert, Mapping) and alert.get("fingerprint"): return { **dict(alert), "completed_at": receipt.get("completed_at") or receipt.get("started_at"), } return {} def build_source_reconciliation( *, sources: Iterable[Mapping[str, object]], provider_attempts: Iterable[Mapping[str, object]], prior_receipts: Iterable[Mapping[str, object]], freshness: Mapping[str, object], now: datetime, retry_interval_minutes: int = 30, max_failure_backoff_minutes: int = 240, alert_cooldown_minutes: int = 360, ) -> Dict[str, object]: """Build a public-safe decision without exposing provider configuration values.""" source_list = [dict(source) for source in sources] attempt_by_source = { str(item.get("source")): dict(item) for item in provider_attempts if item.get("source") } receipt_list = list(prior_receipts) source_states: List[Dict[str, object]] = [] for source in source_list: source_id = str(source.get("id") or "unknown") attempt = attempt_by_source.get(source_id) category = _provider_category(source, attempt) history = _history_attempts(receipt_list, source_id) historical_categories = [ _provider_category(source, historical_attempt) for historical_attempt in history ] current_and_history = [category, *historical_categories] consecutive_empty = _consecutive_count(current_and_history, "empty") consecutive_failures = _consecutive_count(current_and_history, "failed") retry_minutes = None next_retry_at = None if bool(source.get("enabled")) and category in {"duplicate", "empty", "failed", "not_ready"}: retry_minutes = max(1, retry_interval_minutes) if category in {"failed", "not_ready"}: failure_run = max(consecutive_failures, 1) retry_minutes = min( retry_minutes * (2 ** min(failure_run - 1, 3)), max_failure_backoff_minutes, ) next_retry_at = (now + timedelta(minutes=retry_minutes)).isoformat() action = { "disabled": "activate_when_authorized_configuration_is_ready", "not_probed": "probe_on_next_scheduler_run", "not_ready": "reconcile_provider_then_retry", "failed": "retry_with_bounded_backoff_and_probe_fallback", "duplicate": "retry_on_schedule_and_probe_fallback", "empty": "retry_on_schedule_and_probe_fallback", "candidate_found": "validate_and_import_candidate", "imported": "verify_freshness_and_monitor", "ready": "monitor_on_schedule", }[category] source_states.append({ "id": source_id, "enabled": bool(source.get("enabled")), "ready": bool(source.get("ready")), "configuration_status": source.get("status") or "unknown", "runtime_state": category, "error_kind": attempt.get("error_kind") if attempt else None, "candidate_count": int((attempt or {}).get("candidate_count") or 0), "imported_count": int((attempt or {}).get("imported_count") or 0), "duplicate_count": int((attempt or {}).get("duplicate_count") or 0), "last_attempt_at": ( now.isoformat() if attempt and category != "disabled" else history[0].get("attempted_at") if history else None ), "last_success_at": ( now.isoformat() if category == "imported" else next( ( historical_attempt.get("attempted_at") for historical_attempt in history if _provider_category(source, historical_attempt) == "imported" ), None, ) ), "consecutive_empty_runs": consecutive_empty, "consecutive_failure_runs": consecutive_failures, "retry_minutes": retry_minutes, "next_retry_at": next_retry_at, "action": action, }) enabled_states = [item for item in source_states if item["enabled"]] ready_states = [item for item in source_states if item["ready"]] due_times = [ str(item["next_retry_at"]) for item in source_states if item.get("next_retry_at") ] decision_ready = bool(freshness.get("decision_ready")) any_candidate = any( item["runtime_state"] in {"candidate_found", "imported"} for item in source_states ) if decision_ready: status = "closed" automation_decision = "monitor_scheduled_sources" safe_next_action = "none" elif any_candidate: status = "partial" automation_decision = "verify_candidate_import_and_freshness" safe_next_action = "post_import_freshness_verification" elif ready_states: status = "degraded" automation_decision = "retry_ready_sources_and_probe_enabled_fallbacks" safe_next_action = "scheduled_retry_and_enabled_fallback_probe" elif enabled_states: status = "blocked" automation_decision = "repair_enabled_sources_with_bounded_retry" safe_next_action = "reconcile_enabled_source_health_then_retry" else: status = "blocked" automation_decision = "reconcile_authorized_source_configuration" safe_next_action = "reconcile_authorized_source_configuration" lag_bucket = _lag_bucket(freshness.get("data_lag_days")) severity = ( "critical" if lag_bucket in {"critical", "missing"} or not ready_states else "warning" if lag_bucket in {"warning", "stale"} else "info" ) fingerprint_payload = { "lag_bucket": lag_bucket, "source_states": [ [item["id"], item["runtime_state"], item.get("error_kind")] for item in source_states ], } fingerprint = hashlib.sha256( json.dumps(fingerprint_payload, sort_keys=True, ensure_ascii=True).encode("utf-8") ).hexdigest()[:20] previous_alert = _previous_alert(receipt_list) previous_at = _parse_datetime(previous_alert.get("completed_at")) elapsed_minutes = None if previous_at: comparable_now = now if comparable_now.tzinfo and not previous_at.tzinfo: previous_at = previous_at.replace(tzinfo=comparable_now.tzinfo) elif previous_at.tzinfo and not comparable_now.tzinfo: comparable_now = comparable_now.replace(tzinfo=previous_at.tzinfo) elapsed_minutes = max(0, int((comparable_now - previous_at).total_seconds() // 60)) state_changed = previous_alert.get("fingerprint") != fingerprint send_notification = severity in {"warning", "critical"} and ( state_changed or elapsed_minutes is None or elapsed_minutes >= max(1, alert_cooldown_minutes) ) notification_policy = { "event": "pchome_sales_source_freshness", "severity": severity, "fingerprint": fingerprint, "send": send_notification, "reason": ( "state_changed" if state_changed else "cooldown_elapsed" if send_notification else "duplicate_within_cooldown" ), "cooldown_minutes": max(1, alert_cooldown_minutes), } return { "contract": "pchome_sales_source_reconciliation_v1", "status": status, "automation_decision": automation_decision, "manual_review_required": False, "freshness": dict(freshness), "source_priority": [ source_id for source_id in SOURCE_PRIORITY if any(item["id"] == source_id for item in source_states) ], "fallback_chain": [item["id"] for item in ready_states], "single_source_dependency": len(ready_states) == 1, "coverage": { "ready": len(ready_states), "enabled": len(enabled_states), "total": len(source_states), }, "sources": source_states, "next_retry_at": min(due_times) if due_times else None, "safe_next_action": safe_next_action, "notification_policy": notification_policy, }