440 lines
16 KiB
Python
440 lines
16 KiB
Python
"""Controlled runtime and receipt orchestration for Yahoo public offers."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any, Mapping
|
||
|
||
from services.yahoo_shopping_public_source_adapter import (
|
||
ADAPTER_VERSION,
|
||
CONNECTOR_KEY,
|
||
DEFAULT_RECEIPT_ROOT,
|
||
INGESTION_METHOD,
|
||
PLATFORM_CODE,
|
||
POLICY,
|
||
SOURCE_CODE,
|
||
SOURCE_LABEL,
|
||
WORK_ITEM_ID,
|
||
_bounded_int_env,
|
||
_source_contract_guard as _adapter_source_contract_guard,
|
||
enrich_targets_with_pchome_public_evidence,
|
||
parse_pchome_public_product_jsonp,
|
||
parse_yahoo_product_jsonld,
|
||
parse_yahoo_search_html,
|
||
search_yahoo_candidates_for_pchome_products,
|
||
)
|
||
|
||
|
||
_source_contract_guard = _adapter_source_contract_guard
|
||
|
||
|
||
def _no_write_review_sync(
|
||
_engine: Any,
|
||
candidates: list[dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
return {
|
||
"success": True,
|
||
"status": "artifact_candidate_only",
|
||
"candidate_count": len(candidates or []),
|
||
"written_count": 0,
|
||
"writes_database_count": 0,
|
||
"message": "Yahoo 非 exact 候選只保留在 durable receipt，等待下一輪 AI 證據補強。",
|
||
}
|
||
|
||
|
||
def _compact_verified_candidates(run: Mapping[str, Any]) -> list[dict[str, Any]]:
|
||
verification = run.get("candidate_verification")
|
||
verification = verification if isinstance(verification, dict) else {}
|
||
return [
|
||
{
|
||
"target_pchome_product_id": item.get("target_pchome_product_id"),
|
||
"source_product_id": item.get("product_id"),
|
||
"source_product_url": item.get("product_url"),
|
||
"price": item.get("price"),
|
||
"currency": item.get("currency"),
|
||
"match_score": item.get("target_match_score"),
|
||
"candidate_fingerprint": item.get("same_item_candidate_fingerprint"),
|
||
"product_fingerprint_sha256": (item.get("source_provenance") or {}).get(
|
||
"product_fingerprint_sha256"
|
||
),
|
||
}
|
||
for item in verification.get("verified_candidates") or []
|
||
]
|
||
|
||
|
||
def _receipt_summary(data: Mapping[str, Any]) -> dict[str, Any]:
|
||
sync = data.get("external_offer_sync") or {}
|
||
check_mode = sync.get("check_mode_receipt") or {}
|
||
activation = data.get("source_activation") or {}
|
||
rollback = data.get("source_rollback") or {}
|
||
post_verifier = data.get("post_verifier") or {}
|
||
return {
|
||
"target_count": int(data.get("target_count") or 0),
|
||
"source_candidate_count": int(data.get("candidate_count") or 0),
|
||
"verified_candidate_count": int(data.get("auto_compare_count") or 0),
|
||
"review_candidate_count": int(data.get("review_count") or 0),
|
||
"written_count": int(sync.get("written_count") or 0),
|
||
"check_mode_passed": bool(
|
||
check_mode.get("success")
|
||
and int(check_mode.get("selected_count") or 0)
|
||
== int(check_mode.get("candidate_count") or 0)
|
||
),
|
||
"readback_pass_count": int(post_verifier.get("readback_pass_count") or 0),
|
||
"formal_source_activated": bool(activation.get("applied")),
|
||
"writes_database_count": (
|
||
int(sync.get("writes_database_count") or 0)
|
||
+ int(activation.get("writes_database_count") or 0)
|
||
+ int(rollback.get("writes_database_count") or 0)
|
||
),
|
||
}
|
||
|
||
|
||
def _formal_source_state_transition(receipt: Mapping[str, Any]) -> bool | None:
|
||
activation = receipt.get("source_activation")
|
||
activation = activation if isinstance(activation, Mapping) else {}
|
||
rollback = receipt.get("source_rollback")
|
||
rollback = rollback if isinstance(rollback, Mapping) else {}
|
||
|
||
if rollback.get("status") == "rolled_back_and_quarantined" and rollback.get(
|
||
"source_state_restored"
|
||
):
|
||
previous = activation.get("previous_state")
|
||
previous = previous if isinstance(previous, Mapping) else {}
|
||
return bool(
|
||
str(previous.get("status") or "") == "active"
|
||
and previous.get("enabled")
|
||
and previous.get("write_enabled")
|
||
)
|
||
|
||
if activation.get("applied") and activation.get("status") in {
|
||
"activated",
|
||
"already_active_verified",
|
||
}:
|
||
return True
|
||
|
||
summary = receipt.get("summary")
|
||
summary = summary if isinstance(summary, Mapping) else {}
|
||
if summary.get("formal_source_activated") is True:
|
||
return True
|
||
return None
|
||
|
||
|
||
def _write_compact_receipt(
|
||
result: Mapping[str, Any],
|
||
*,
|
||
receipt_root: str | Path,
|
||
) -> dict[str, Any]:
|
||
data = result.get("data") if isinstance(result.get("data"), dict) else {}
|
||
run = data.get("reconciliation_receipt")
|
||
run = run if isinstance(run, dict) else {}
|
||
identity = data.get("identity") if isinstance(data.get("identity"), dict) else {}
|
||
run_id = str(identity.get("run_id") or "unknown")
|
||
summary = _receipt_summary(data)
|
||
receipt = {
|
||
"policy": POLICY,
|
||
"adapter_version": ADAPTER_VERSION,
|
||
"work_item_id": WORK_ITEM_ID,
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"identity": identity,
|
||
"success": bool(result.get("success")),
|
||
"terminal_status": data.get("terminal_status"),
|
||
"next_machine_action": data.get("next_machine_action"),
|
||
"source_receipt": run.get("source_receipt") or {},
|
||
"summary": summary,
|
||
"verified_candidates": _compact_verified_candidates(run),
|
||
"source_activation": data.get("source_activation") or {},
|
||
"source_rollback": data.get("source_rollback") or {},
|
||
"post_verifier": data.get("post_verifier") or {},
|
||
"controlled_apply": {
|
||
"public_network_read": True,
|
||
"artifact_write": True,
|
||
"database_write": bool(summary["writes_database_count"]),
|
||
"check_mode_passed": bool(summary["check_mode_passed"]),
|
||
"secret_read": False,
|
||
"cookie_or_session_read": False,
|
||
"login_allowed": False,
|
||
"pixelrag_direct_price_write": False,
|
||
"independent_same_item_verifier_required": True,
|
||
"source_activation_canary_required": True,
|
||
},
|
||
"learning_writeback": {
|
||
"reconciliation_receipt_persisted": bool(
|
||
(data.get("receipt_persistence") or {}).get("success")
|
||
),
|
||
"artifact_receipt_persisted": True,
|
||
},
|
||
}
|
||
root = Path(receipt_root)
|
||
root.mkdir(parents=True, exist_ok=True)
|
||
target = root / f"{run_id}.json"
|
||
temporary = target.with_suffix(".json.tmp")
|
||
temporary.write_text(
|
||
json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True),
|
||
encoding="utf-8",
|
||
)
|
||
os.replace(temporary, target)
|
||
persisted = json.loads(target.read_text(encoding="utf-8"))
|
||
return {
|
||
"success": persisted.get("identity") == identity,
|
||
"status": "persisted_and_verified",
|
||
"receipt_path": str(target),
|
||
"receipt_sha256": hashlib.sha256(target.read_bytes()).hexdigest(),
|
||
"summary": summary,
|
||
}
|
||
|
||
|
||
def run_pchome_growth_yahoo_backfill(
|
||
engine: Any,
|
||
*,
|
||
limit: int = 12,
|
||
request_get=None,
|
||
receipt_root: str | Path | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Run the bounded Yahoo detect-to-verify-to-activate production loop."""
|
||
from services.external_marketplace_canary_service import (
|
||
activate_external_market_source_after_canary,
|
||
rollback_external_market_source_canary,
|
||
sync_targeted_marketplace_candidates_to_external_offers,
|
||
)
|
||
from services.pchome_growth_momo_backfill_service import (
|
||
run_pchome_growth_momo_backfill,
|
||
)
|
||
from services.pchome_growth_same_item_reconciliation import (
|
||
readback_external_offer_candidates,
|
||
)
|
||
|
||
def search(targets: list[dict[str, Any]], bounded_limit: int):
|
||
return search_yahoo_candidates_for_pchome_products(
|
||
targets,
|
||
bounded_limit,
|
||
request_get=request_get,
|
||
)
|
||
|
||
def sync(sync_engine: Any, candidates: list[dict[str, Any]]):
|
||
sync_kwargs = {
|
||
"source_code": SOURCE_CODE,
|
||
"platform_code": PLATFORM_CODE,
|
||
"ingestion_method": INGESTION_METHOD,
|
||
"connector_key": CONNECTOR_KEY,
|
||
"source_label": SOURCE_LABEL,
|
||
"ttl_hours": _bounded_int_env(
|
||
"PCHOME_GROWTH_YAHOO_OFFER_TTL_HOURS",
|
||
30,
|
||
6,
|
||
72,
|
||
),
|
||
}
|
||
check_mode = sync_targeted_marketplace_candidates_to_external_offers(
|
||
sync_engine,
|
||
candidates,
|
||
dry_run=True,
|
||
**sync_kwargs,
|
||
)
|
||
if not check_mode.get("success") or int(
|
||
check_mode.get("selected_count") or 0
|
||
) != len(candidates):
|
||
return {
|
||
"success": False,
|
||
"status": "check_mode_failed_no_write",
|
||
"candidate_count": len(candidates),
|
||
"written_count": 0,
|
||
"writes_database_count": 0,
|
||
"check_mode_receipt": check_mode,
|
||
"message": "Yahoo check-mode 未完整通過，本輪已停止且沒有寫入。",
|
||
}
|
||
applied = sync_targeted_marketplace_candidates_to_external_offers(
|
||
sync_engine,
|
||
candidates,
|
||
dry_run=False,
|
||
**sync_kwargs,
|
||
)
|
||
applied["check_mode_receipt"] = check_mode
|
||
return applied
|
||
|
||
def readback(readback_engine: Any, candidates: list[dict[str, Any]]):
|
||
return readback_external_offer_candidates(
|
||
readback_engine,
|
||
candidates,
|
||
source_code=SOURCE_CODE,
|
||
ingestion_method=INGESTION_METHOD,
|
||
)
|
||
|
||
def activate(
|
||
activation_engine: Any,
|
||
*,
|
||
candidates: list[dict[str, Any]],
|
||
independent_readback: dict[str, Any],
|
||
identity: dict[str, str],
|
||
):
|
||
return activate_external_market_source_after_canary(
|
||
activation_engine,
|
||
source_code=SOURCE_CODE,
|
||
candidates=candidates,
|
||
independent_readback=independent_readback,
|
||
identity=identity,
|
||
minimum_verified_count=_bounded_int_env(
|
||
"PCHOME_GROWTH_YAHOO_ACTIVATION_MIN_VERIFIED",
|
||
2,
|
||
2,
|
||
10,
|
||
),
|
||
)
|
||
|
||
def rollback(
|
||
rollback_engine: Any,
|
||
*,
|
||
candidates: list[dict[str, Any]],
|
||
identity: dict[str, str],
|
||
activation_receipt: dict[str, Any],
|
||
):
|
||
return rollback_external_market_source_canary(
|
||
rollback_engine,
|
||
source_code=SOURCE_CODE,
|
||
ingestion_method=INGESTION_METHOD,
|
||
candidates=candidates,
|
||
identity=identity,
|
||
activation_receipt=activation_receipt,
|
||
)
|
||
|
||
result = run_pchome_growth_momo_backfill(
|
||
engine,
|
||
limit=limit,
|
||
search_func=search,
|
||
sync_func=sync,
|
||
sync_review_func=_no_write_review_sync,
|
||
readback_func=readback,
|
||
source_profile={
|
||
"display_name": SOURCE_LABEL,
|
||
"receipt_source": CONNECTOR_KEY,
|
||
"work_item_id": WORK_ITEM_ID,
|
||
"formal_compare_types": ["total_price"],
|
||
},
|
||
source_activation_func=activate,
|
||
source_rollback_func=rollback,
|
||
)
|
||
try:
|
||
artifact = _write_compact_receipt(
|
||
result,
|
||
receipt_root=receipt_root or DEFAULT_RECEIPT_ROOT,
|
||
)
|
||
except Exception as exc:
|
||
artifact = {
|
||
"success": False,
|
||
"status": "artifact_write_failed",
|
||
"error_type": type(exc).__name__,
|
||
"error": str(exc)[:200],
|
||
}
|
||
result["success"] = False
|
||
result.setdefault("data", {})["artifact_receipt"] = artifact
|
||
return result
|
||
|
||
|
||
def build_yahoo_shopping_runtime_readback(
|
||
*,
|
||
receipt_root: str | Path | None = None,
|
||
max_age_hours: int = 30,
|
||
) -> dict[str, Any]:
|
||
"""Read the latest public-safe Yahoo runtime receipt without network or DB calls."""
|
||
root = Path(receipt_root or DEFAULT_RECEIPT_ROOT)
|
||
paths = sorted(
|
||
root.glob("*.json") if root.exists() else [],
|
||
key=lambda path: path.stat().st_mtime,
|
||
reverse=True,
|
||
)
|
||
if not paths:
|
||
return {
|
||
"success": True,
|
||
"status": "not_started",
|
||
"source_code": SOURCE_CODE,
|
||
"has_candidate_runtime_data": False,
|
||
"candidate_offer_count": 0,
|
||
"receipt_count": 0,
|
||
"writes_database_count": 0,
|
||
}
|
||
latest_path = paths[0]
|
||
try:
|
||
receipt = json.loads(latest_path.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError) as exc:
|
||
return {
|
||
"success": False,
|
||
"status": "invalid_receipt",
|
||
"source_code": SOURCE_CODE,
|
||
"has_candidate_runtime_data": False,
|
||
"candidate_offer_count": 0,
|
||
"receipt_count": len(paths),
|
||
"error_type": type(exc).__name__,
|
||
"writes_database_count": 0,
|
||
}
|
||
generated_at = str(receipt.get("generated_at") or "")
|
||
try:
|
||
generated = datetime.fromisoformat(generated_at.replace("Z", "+00:00"))
|
||
if generated.tzinfo is None:
|
||
generated = generated.replace(tzinfo=timezone.utc)
|
||
age_hours = max(
|
||
0.0,
|
||
(
|
||
datetime.now(timezone.utc) - generated.astimezone(timezone.utc)
|
||
).total_seconds()
|
||
/ 3600,
|
||
)
|
||
except ValueError:
|
||
age_hours = None
|
||
stale = age_hours is None or age_hours > max(
|
||
1,
|
||
min(int(max_age_hours or 30), 168),
|
||
)
|
||
summary = receipt.get("summary")
|
||
summary = summary if isinstance(summary, dict) else {}
|
||
formal_source_activated = None
|
||
for path in paths:
|
||
try:
|
||
historical = json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError):
|
||
continue
|
||
formal_source_activated = _formal_source_state_transition(historical)
|
||
if formal_source_activated is not None:
|
||
break
|
||
if formal_source_activated is None:
|
||
formal_source_activated = bool(summary.get("formal_source_activated"))
|
||
candidate_count = int(summary.get("source_candidate_count") or 0)
|
||
has_candidate = bool(candidate_count and not stale)
|
||
return {
|
||
"success": True,
|
||
"status": "ready" if has_candidate else "stale" if stale else "no_candidate",
|
||
"policy": receipt.get("policy"),
|
||
"adapter_version": receipt.get("adapter_version"),
|
||
"source_code": SOURCE_CODE,
|
||
"platform_code": PLATFORM_CODE,
|
||
"has_candidate_runtime_data": has_candidate,
|
||
"candidate_offer_count": candidate_count if has_candidate else 0,
|
||
"verified_candidate_count": int(summary.get("verified_candidate_count") or 0),
|
||
"written_count": int(summary.get("written_count") or 0),
|
||
"formal_source_activated": formal_source_activated,
|
||
"last_seen_at": generated_at,
|
||
"age_hours": round(age_hours, 3) if age_hours is not None else None,
|
||
"stale": stale,
|
||
"receipt_count": len(paths),
|
||
"receipt_path": str(latest_path),
|
||
"terminal_status": receipt.get("terminal_status"),
|
||
"next_machine_action": receipt.get("next_machine_action"),
|
||
"writes_database_count": 0,
|
||
}
|
||
|
||
|
||
__all__ = [
|
||
"ADAPTER_VERSION",
|
||
"DEFAULT_RECEIPT_ROOT",
|
||
"POLICY",
|
||
"build_yahoo_shopping_runtime_readback",
|
||
"enrich_targets_with_pchome_public_evidence",
|
||
"parse_pchome_public_product_jsonp",
|
||
"parse_yahoo_product_jsonld",
|
||
"parse_yahoo_search_html",
|
||
"run_pchome_growth_yahoo_backfill",
|
||
"search_yahoo_candidates_for_pchome_products",
|
||
]
|