feat(growth): automate sales source reconciliation

services/import_service.py

@@ -1175,7 +1175,7 @@ class ImportService:
             logger.error(traceback.format_exc())
             return False
 
-    def auto_import_from_drive(self) -> Dict[str, Any]:
+    def auto_import_from_drive(self, *, send_stale_alert: bool = True) -> Dict[str, Any]:
         """
         從 Google Drive 自動匯入檔案
 
@@ -1243,7 +1243,7 @@ class ImportService:
                         days_since = (datetime.now(TAIPEI_TZ).date() - normalized_last_date).days
                         data_lag_days = days_since
                         latest_sales_date = str(normalized_last_date)
-                        if days_since >= 3:
+                        if days_since >= 3 and send_stale_alert:
                             _send_data_stale_alert(
                                 report_type="upstream_drive",
                                 last_date=str(normalized_last_date),

services/pchome_sales_acquisition_service.py

@@ -23,6 +23,7 @@ from services.pchome_sales_acquisition_providers import (
     env_bool,
     env_int,
 )
+from services.pchome_sales_source_reconciliation import build_source_reconciliation
 
 
 logger = logging.getLogger(__name__)
@@ -80,7 +81,20 @@ class PChomeSalesAcquisitionService:
         finally:
             session.close()
 
-    def readiness(self) -> Dict[str, object]:
+    def _recent_receipts(self, limit: int = 48) -> List[Dict[str, object]]:
+        session = Session()
+        try:
+            receipts = session.query(SalesAcquisitionReceipt).order_by(
+                SalesAcquisitionReceipt.started_at.desc()
+            ).limit(max(1, min(limit, 336))).all()
+            return [receipt.to_public_dict() for receipt in receipts]
+        except Exception:
+            logger.warning("Unable to read sales acquisition history", exc_info=True)
+            return []
+        finally:
+            session.close()
+
+    def _source_inventory(self) -> List[Dict[str, object]]:
         try:
             drive = drive_service.check_auth_readiness(refresh_expired=False)
             drive_ready = bool(drive.get("ready"))
@@ -104,7 +118,7 @@ class PChomeSalesAcquisitionService:
         )
         local_path = os.getenv("PCHOME_SALES_LOCAL_DROP_DIR", "").strip()
         local_ready = bool(local_path and Path(local_path).is_dir())
-        sources = [
+        return [
             {"id": "google_drive", "enabled": True, "ready": drive_ready, "status": drive_status},
             {
                 "id": "authorized_https",
@@ -125,8 +139,13 @@ class PChomeSalesAcquisitionService:
                 "status": "ready" if local_ready else "disabled" if not local_path else "path_missing",
             },
         ]
+
+    def readiness(self) -> Dict[str, object]:
+        sources = self._source_inventory()
         ready_count = sum(1 for source in sources if source["ready"])
         freshness = self._freshness()
+        last_receipt = self._latest_receipt()
+        last_stages = (last_receipt or {}).get("stages") or {}
         return {
             "success": True,
             "automation_mode": "controlled_apply",
@@ -141,7 +160,8 @@ class PChomeSalesAcquisitionService:
             },
             "freshness": freshness,
             "runtime_closure": "closed" if freshness["decision_ready"] else "blocked_by_upstream_freshness",
-            "last_receipt": self._latest_receipt(),
+            "source_reconciliation": last_stages.get("source_reconciliation"),
+            "last_receipt": last_receipt,
         }
 
     def _persist_receipt(
@@ -206,6 +226,9 @@ class PChomeSalesAcquisitionService:
         span_id = uuid.uuid4().hex[:16]
         run_id = uuid.uuid4().hex
         before = self._freshness()
+        source_inventory = self._source_inventory()
+        source_by_id = {str(source["id"]): source for source in source_inventory}
+        prior_receipts = self._recent_receipts()
         stages: Dict[str, object] = {
             "sensor": {"status": "started", "trigger": trigger, "trace_id": trace_id, "span_id": span_id},
             "normalize": {"asset_id": "pchome.daily_sales.authorized_report"},
@@ -224,9 +247,11 @@ class PChomeSalesAcquisitionService:
         error_kind = None
 
         try:
-            drive_result = import_service.auto_import_from_drive()
+            drive_result = import_service.auto_import_from_drive(send_stale_alert=False)
             stages["providers"].append({
                 "source": "google_drive",
+                "enabled": True,
+                "ready": bool(source_by_id.get("google_drive", {}).get("ready")),
                 "status": drive_result.get("status") or ("completed" if drive_result.get("success") else "failed"),
                 "file_count": drive_result.get("file_count", 0),
                 "imported_count": drive_result.get("imported_count", 0),
@@ -258,17 +283,49 @@ class PChomeSalesAcquisitionService:
                     ("authorized_imap", self.providers.imap_candidates),
                     ("controlled_local_drop", self.providers.local_candidates),
                 ):
+                    source_state = source_by_id.get(source_name) or {}
+                    if not source_state.get("enabled"):
+                        stages["providers"].append({
+                            "source": source_name,
+                            "enabled": False,
+                            "ready": False,
+                            "status": "disabled",
+                            "candidate_count": 0,
+                        })
+                        continue
+                    if not source_state.get("ready"):
+                        failed_results.append({
+                            "source_type": source_name,
+                            "error_kind": "provider_not_ready",
+                            "message": "授權來源設定尚未達到自動執行條件。",
+                        })
+                        stages["providers"].append({
+                            "source": source_name,
+                            "enabled": True,
+                            "ready": False,
+                            "status": "not_ready",
+                            "error_kind": "provider_not_ready",
+                        })
+                        continue
                     try:
                         provider_candidates = provider()
                         candidates.extend(provider_candidates)
                         stages["providers"].append({
                             "source": source_name,
+                            "enabled": True,
+                            "ready": True,
                             "status": "candidate_found" if provider_candidates else "no_candidate",
                             "candidate_count": len(provider_candidates),
                         })
                     except ProviderError as exc:
                         failed_results.append({"source_type": source_name, "error_kind": exc.kind, "message": exc.public_message})
-                        stages["providers"].append({"source": source_name, "status": "failed", "error_kind": exc.kind})
+                        stages["providers"].append({
+                            "source": source_name,
+                            "enabled": True,
+                            "ready": True,
+                            "status": "failed",
+                            "error_kind": exc.kind,
+                        })
 
                 max_files = env_int("PCHOME_SALES_MAX_FILES_PER_RUN", 5, 1, 25)
                 for candidate in candidates[:max_files]:
@@ -342,6 +399,36 @@ class PChomeSalesAcquisitionService:
                 except Exception:
                     logger.warning("Unable to clean sales acquisition temp file", exc_info=True)
 
+        for provider_stage in stages["providers"]:
+            provider_source = provider_stage.get("source")
+            imported_for_source = [
+                item for item in successful_results
+                if item.get("source_type") == provider_source
+            ]
+            duplicates_for_source = [
+                item for item in duplicate_results
+                if item.get("source_type") == provider_source
+            ]
+            failures_for_source = [
+                item for item in failed_results
+                if item.get("source_type") == provider_source
+            ]
+            if imported_for_source:
+                provider_stage["status"] = "completed"
+                provider_stage["imported_count"] = len(imported_for_source)
+                provider_stage["rows_imported"] = sum(
+                    int(item.get("rows") or 0) for item in imported_for_source
+                )
+            elif duplicates_for_source:
+                provider_stage["status"] = "duplicate"
+                provider_stage["duplicate_count"] = len(duplicates_for_source)
+            elif failures_for_source and provider_stage.get("status") != "disabled":
+                provider_stage["status"] = "failed"
+                provider_stage["error_kind"] = (
+                    provider_stage.get("error_kind")
+                    or failures_for_source[0].get("error_kind")
+                )
+
         verifier = self._freshness()
         rows_imported = sum(int(item.get("rows") or 0) for item in successful_results)
         import_job_id = next((item.get("job_id") for item in reversed(successful_results) if item.get("job_id")), None)
@@ -409,6 +496,24 @@ class PChomeSalesAcquisitionService:
             decision = "no_authorized_candidate"
             message = "所有已啟用授權來源目前都沒有新業績檔，正式資料未變更。"
 
+        source_reconciliation = build_source_reconciliation(
+            sources=source_inventory,
+            provider_attempts=stages["providers"],
+            prior_receipts=prior_receipts,
+            freshness=verifier,
+            now=datetime.now(TAIPEI_TZ),
+            retry_interval_minutes=env_int(
+                "PCHOME_SALES_RETRY_INTERVAL_MINUTES", 30, 1, 1440
+            ),
+            max_failure_backoff_minutes=env_int(
+                "PCHOME_SALES_MAX_FAILURE_BACKOFF_MINUTES", 240, 30, 1440
+            ),
+            alert_cooldown_minutes=env_int(
+                "PCHOME_SALES_ALERT_COOLDOWN_MINUTES", 360, 30, 1440
+            ),
+        )
+        stages["source_reconciliation"] = source_reconciliation
+
         stages["execution"] = {
             "status": status,
             "successful_files": len(successful_results),
@@ -458,6 +563,13 @@ class PChomeSalesAcquisitionService:
             decision = "runtime_verified_receipt_write_failed"
             message = "業績資料已通過驗證，但治理 receipt 寫入失敗，閉環尚未完成。"
 
+        if error_kind == "source_finalize_failed":
+            safe_next_action = "retry_source_finalization"
+        elif status == "failed_rolled_back":
+            safe_next_action = "replay_transaction_after_rollback"
+        else:
+            safe_next_action = source_reconciliation.get("safe_next_action") or "none"
+
         return {
             "success": status in {"completed", "completed_no_write"},
             "status": status,
@@ -481,15 +593,10 @@ class PChomeSalesAcquisitionService:
             "decision_ready": verifier.get("decision_ready"),
             "requires_upstream_acquisition": not bool(verifier.get("decision_ready")),
             "manual_review_required": False,
-            "safe_next_action": (
-                "scheduler_retry_authorized_sources"
-                if status == "blocked_with_safe_next_action"
-                else "repair_provider_or_replay_transaction"
-                if status in {"failed_rolled_back", "degraded_no_write"}
-                else "retry_source_finalization"
-                if error_kind == "source_finalize_failed"
-                else "none"
-            ),
+            "safe_next_action": safe_next_action,
+            "next_retry_at": source_reconciliation.get("next_retry_at"),
+            "notification_policy": source_reconciliation.get("notification_policy"),
+            "source_reconciliation": source_reconciliation,
             "providers": stages["providers"],
             "verifier": verifier,
         }

services/pchome_sales_source_reconciliation.py

@@ -0,0 +1,326 @@
+"""Cross-run source health, retry, fallback, and alert decisions for PChome sales."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timedelta
+from typing import Dict, Iterable, List, Mapping, Optional
+
+
+SOURCE_PRIORITY = (
+    "google_drive",
+    "authorized_https",
+    "authorized_imap",
+    "controlled_local_drop",
+)
+
+_EMPTY_STATUSES = {
+    "no_candidate",
+    "no_pending_file",
+    "upstream_missing",
+    "upstream_stale",
+}
+_FAILED_STATUSES = {"failed", "not_ready", "probe_failed"}
+
+
+def _provider_category(
+    source: Mapping[str, object],
+    attempt: Optional[Mapping[str, object]],
+) -> str:
+    if not bool(source.get("enabled")):
+        return "disabled"
+    if not attempt:
+        return "not_probed"
+
+    status = str(attempt.get("status") or "unknown")
+    if int(attempt.get("imported_count") or 0) > 0:
+        return "imported"
+    if int(attempt.get("duplicate_count") or 0) > 0 or status == "duplicate":
+        return "duplicate"
+    if int(attempt.get("candidate_count") or 0) > 0:
+        return "candidate_found"
+    if attempt.get("error_kind") or status in _FAILED_STATUSES:
+        return "failed"
+    if status in _EMPTY_STATUSES:
+        return "empty"
+    if not bool(source.get("ready")):
+        return "not_ready"
+    return "ready"
+
+
+def _history_attempts(
+    receipts: Iterable[Mapping[str, object]],
+    source_id: str,
+) -> List[Dict[str, object]]:
+    attempts: List[Dict[str, object]] = []
+    for receipt in receipts:
+        stages = receipt.get("stages") or {}
+        if not isinstance(stages, Mapping):
+            continue
+        providers = stages.get("providers") or []
+        if not isinstance(providers, list):
+            continue
+        for provider in providers:
+            if isinstance(provider, Mapping) and provider.get("source") == source_id:
+                attempts.append({
+                    **dict(provider),
+                    "receipt_status": receipt.get("status"),
+                    "attempted_at": receipt.get("completed_at") or receipt.get("started_at"),
+                })
+                break
+    return attempts
+
+
+def _consecutive_count(categories: Iterable[str], target: str) -> int:
+    count = 0
+    for category in categories:
+        if category != target:
+            break
+        count += 1
+    return count
+
+
+def _parse_datetime(value: object) -> Optional[datetime]:
+    if not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except (TypeError, ValueError):
+        return None
+    return parsed
+
+
+def _lag_bucket(lag_days: object) -> str:
+    if lag_days is None:
+        return "missing"
+    try:
+        lag = int(lag_days)
+    except (TypeError, ValueError):
+        return "missing"
+    if lag <= 1:
+        return "fresh"
+    if lag <= 3:
+        return "warning"
+    if lag <= 7:
+        return "stale"
+    return "critical"
+
+
+def _previous_alert(receipts: Iterable[Mapping[str, object]]) -> Dict[str, object]:
+    for receipt in receipts:
+        stages = receipt.get("stages") or {}
+        if not isinstance(stages, Mapping):
+            continue
+        reconciliation = stages.get("source_reconciliation") or {}
+        if not isinstance(reconciliation, Mapping):
+            continue
+        alert = reconciliation.get("notification_policy") or {}
+        if isinstance(alert, Mapping) and alert.get("fingerprint"):
+            return {
+                **dict(alert),
+                "completed_at": receipt.get("completed_at") or receipt.get("started_at"),
+            }
+    return {}
+
+
+def build_source_reconciliation(
+    *,
+    sources: Iterable[Mapping[str, object]],
+    provider_attempts: Iterable[Mapping[str, object]],
+    prior_receipts: Iterable[Mapping[str, object]],
+    freshness: Mapping[str, object],
+    now: datetime,
+    retry_interval_minutes: int = 30,
+    max_failure_backoff_minutes: int = 240,
+    alert_cooldown_minutes: int = 360,
+) -> Dict[str, object]:
+    """Build a public-safe decision without exposing provider configuration values."""
+    source_list = [dict(source) for source in sources]
+    attempt_by_source = {
+        str(item.get("source")): dict(item)
+        for item in provider_attempts
+        if item.get("source")
+    }
+    receipt_list = list(prior_receipts)
+    source_states: List[Dict[str, object]] = []
+
+    for source in source_list:
+        source_id = str(source.get("id") or "unknown")
+        attempt = attempt_by_source.get(source_id)
+        category = _provider_category(source, attempt)
+        history = _history_attempts(receipt_list, source_id)
+        historical_categories = [
+            _provider_category(source, historical_attempt)
+            for historical_attempt in history
+        ]
+        current_and_history = [category, *historical_categories]
+        consecutive_empty = _consecutive_count(current_and_history, "empty")
+        consecutive_failures = _consecutive_count(current_and_history, "failed")
+
+        retry_minutes = None
+        next_retry_at = None
+        if bool(source.get("enabled")) and category in {"duplicate", "empty", "failed", "not_ready"}:
+            retry_minutes = max(1, retry_interval_minutes)
+            if category in {"failed", "not_ready"}:
+                failure_run = max(consecutive_failures, 1)
+                retry_minutes = min(
+                    retry_minutes * (2 ** min(failure_run - 1, 3)),
+                    max_failure_backoff_minutes,
+                )
+            next_retry_at = (now + timedelta(minutes=retry_minutes)).isoformat()
+
+        action = {
+            "disabled": "activate_when_authorized_configuration_is_ready",
+            "not_probed": "probe_on_next_scheduler_run",
+            "not_ready": "reconcile_provider_then_retry",
+            "failed": "retry_with_bounded_backoff_and_probe_fallback",
+            "duplicate": "retry_on_schedule_and_probe_fallback",
+            "empty": "retry_on_schedule_and_probe_fallback",
+            "candidate_found": "validate_and_import_candidate",
+            "imported": "verify_freshness_and_monitor",
+            "ready": "monitor_on_schedule",
+        }[category]
+        source_states.append({
+            "id": source_id,
+            "enabled": bool(source.get("enabled")),
+            "ready": bool(source.get("ready")),
+            "configuration_status": source.get("status") or "unknown",
+            "runtime_state": category,
+            "error_kind": attempt.get("error_kind") if attempt else None,
+            "candidate_count": int((attempt or {}).get("candidate_count") or 0),
+            "imported_count": int((attempt or {}).get("imported_count") or 0),
+            "duplicate_count": int((attempt or {}).get("duplicate_count") or 0),
+            "last_attempt_at": (
+                now.isoformat()
+                if attempt and category != "disabled"
+                else history[0].get("attempted_at")
+                if history
+                else None
+            ),
+            "last_success_at": (
+                now.isoformat()
+                if category == "imported"
+                else next(
+                    (
+                        historical_attempt.get("attempted_at")
+                        for historical_attempt in history
+                        if _provider_category(source, historical_attempt) == "imported"
+                    ),
+                    None,
+                )
+            ),
+            "consecutive_empty_runs": consecutive_empty,
+            "consecutive_failure_runs": consecutive_failures,
+            "retry_minutes": retry_minutes,
+            "next_retry_at": next_retry_at,
+            "action": action,
+        })
+
+    enabled_states = [item for item in source_states if item["enabled"]]
+    ready_states = [item for item in source_states if item["ready"]]
+    due_times = [
+        str(item["next_retry_at"])
+        for item in source_states
+        if item.get("next_retry_at")
+    ]
+    decision_ready = bool(freshness.get("decision_ready"))
+    any_candidate = any(
+        item["runtime_state"] in {"candidate_found", "imported"}
+        for item in source_states
+    )
+
+    if decision_ready:
+        status = "closed"
+        automation_decision = "monitor_scheduled_sources"
+        safe_next_action = "none"
+    elif any_candidate:
+        status = "partial"
+        automation_decision = "verify_candidate_import_and_freshness"
+        safe_next_action = "post_import_freshness_verification"
+    elif ready_states:
+        status = "degraded"
+        automation_decision = "retry_ready_sources_and_probe_enabled_fallbacks"
+        safe_next_action = "scheduled_retry_and_enabled_fallback_probe"
+    elif enabled_states:
+        status = "blocked"
+        automation_decision = "repair_enabled_sources_with_bounded_retry"
+        safe_next_action = "reconcile_enabled_source_health_then_retry"
+    else:
+        status = "blocked"
+        automation_decision = "reconcile_authorized_source_configuration"
+        safe_next_action = "reconcile_authorized_source_configuration"
+
+    lag_bucket = _lag_bucket(freshness.get("data_lag_days"))
+    severity = (
+        "critical"
+        if lag_bucket in {"critical", "missing"} or not ready_states
+        else "warning"
+        if lag_bucket in {"warning", "stale"}
+        else "info"
+    )
+    fingerprint_payload = {
+        "lag_bucket": lag_bucket,
+        "source_states": [
+            [item["id"], item["runtime_state"], item.get("error_kind")]
+            for item in source_states
+        ],
+    }
+    fingerprint = hashlib.sha256(
+        json.dumps(fingerprint_payload, sort_keys=True, ensure_ascii=True).encode("utf-8")
+    ).hexdigest()[:20]
+    previous_alert = _previous_alert(receipt_list)
+    previous_at = _parse_datetime(previous_alert.get("completed_at"))
+    elapsed_minutes = None
+    if previous_at:
+        comparable_now = now
+        if comparable_now.tzinfo and not previous_at.tzinfo:
+            previous_at = previous_at.replace(tzinfo=comparable_now.tzinfo)
+        elif previous_at.tzinfo and not comparable_now.tzinfo:
+            comparable_now = comparable_now.replace(tzinfo=previous_at.tzinfo)
+        elapsed_minutes = max(0, int((comparable_now - previous_at).total_seconds() // 60))
+    state_changed = previous_alert.get("fingerprint") != fingerprint
+    send_notification = severity in {"warning", "critical"} and (
+        state_changed
+        or elapsed_minutes is None
+        or elapsed_minutes >= max(1, alert_cooldown_minutes)
+    )
+
+    notification_policy = {
+        "event": "pchome_sales_source_freshness",
+        "severity": severity,
+        "fingerprint": fingerprint,
+        "send": send_notification,
+        "reason": (
+            "state_changed"
+            if state_changed
+            else "cooldown_elapsed"
+            if send_notification
+            else "duplicate_within_cooldown"
+        ),
+        "cooldown_minutes": max(1, alert_cooldown_minutes),
+    }
+
+    return {
+        "contract": "pchome_sales_source_reconciliation_v1",
+        "status": status,
+        "automation_decision": automation_decision,
+        "manual_review_required": False,
+        "freshness": dict(freshness),
+        "source_priority": [
+            source_id
+            for source_id in SOURCE_PRIORITY
+            if any(item["id"] == source_id for item in source_states)
+        ],
+        "fallback_chain": [item["id"] for item in ready_states],
+        "single_source_dependency": len(ready_states) == 1,
+        "coverage": {
+            "ready": len(ready_states),
+            "enabled": len(enabled_states),
+            "total": len(source_states),
+        },
+        "sources": source_states,
+        "next_retry_at": min(due_times) if due_times else None,
+        "safe_next_action": safe_next_action,
+        "notification_policy": notification_policy,
+    }