566 lines
20 KiB
Python
566 lines
20 KiB
Python
"""Controlled apply, canary activation, and rollback for external offers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
from sqlalchemy import text
|
|
|
|
from services.external_market_offer_service import (
|
|
_ensure_external_market_source_seeds,
|
|
_has_table,
|
|
_load_json_dict,
|
|
_quality_score_from_match,
|
|
_targeted_candidate_auto_type,
|
|
_targeted_candidate_needs_review,
|
|
_targeted_candidate_sync_rank,
|
|
_to_float,
|
|
_upsert_external_offer,
|
|
)
|
|
from services.pchome_growth_cache_state import mark_pchome_growth_cache_stale
|
|
|
|
|
|
def _targeted_marketplace_candidate_to_external_offer(
|
|
candidate: dict[str, Any],
|
|
*,
|
|
source_code: str,
|
|
platform_code: str,
|
|
ingestion_method: str,
|
|
connector_key: str,
|
|
source_label: str,
|
|
observed_at: datetime,
|
|
ttl_hours: int,
|
|
) -> tuple[dict[str, Any] | None, str]:
|
|
"""Normalize an independently verified marketplace candidate for controlled apply."""
|
|
guard = candidate.get("source_contract_guard")
|
|
if not isinstance(guard, dict) or not bool(guard.get("passed")):
|
|
return None, "來源契約或商品頁驗證未通過"
|
|
reconciliation = candidate.get("same_item_reconciliation")
|
|
if not isinstance(reconciliation, dict) or not bool(
|
|
reconciliation.get("independent_verifier_passed")
|
|
):
|
|
return None, "獨立同款 verifier 未通過"
|
|
if _targeted_candidate_auto_type(candidate) != "total_price":
|
|
return None, "目前只允許 exact total-price marketplace candidate"
|
|
if _targeted_candidate_needs_review(candidate):
|
|
return None, "候選仍有規格、款式或促銷條件疑慮"
|
|
|
|
source_product_id = str(
|
|
candidate.get("product_id")
|
|
or candidate.get("goodsCode")
|
|
or candidate.get("id")
|
|
or ""
|
|
).strip()
|
|
pchome_product_id = str(candidate.get("target_pchome_product_id") or "").strip()
|
|
source_price = _to_float(candidate.get("price"))
|
|
pchome_price = _to_float(candidate.get("target_pchome_price"))
|
|
currency = str(candidate.get("currency") or "TWD").upper()
|
|
if not source_product_id:
|
|
return None, f"缺少{source_label}商品 ID"
|
|
if not pchome_product_id:
|
|
return None, "缺少 PChome 商品 ID"
|
|
if not source_price or source_price <= 0:
|
|
return None, f"缺少{source_label}售價"
|
|
if not pchome_price or pchome_price <= 0:
|
|
return None, "缺少 PChome 公開售價"
|
|
if currency != "TWD":
|
|
return None, "目前只允許 TWD 公開報價"
|
|
|
|
quality_score = _quality_score_from_match(candidate.get("target_match_score"))
|
|
if quality_score < 76:
|
|
return None, "同款分數低於自動同步門檻"
|
|
|
|
title = str(
|
|
candidate.get("name") or candidate.get("title") or source_product_id
|
|
).strip()
|
|
provenance = candidate.get("source_provenance")
|
|
provenance = provenance if isinstance(provenance, dict) else {}
|
|
raw_payload = {
|
|
"source": connector_key,
|
|
"source_code": source_code,
|
|
"platform_code": platform_code,
|
|
"auto_compare_type": "total_price",
|
|
"price_basis": "total_price",
|
|
"pchome_public_price": pchome_price,
|
|
"pchome_public_name": candidate.get("target_pchome_name"),
|
|
"pchome_public_evidence": candidate.get("target_public_evidence") or {},
|
|
"match_score": candidate.get("target_match_score"),
|
|
"match_reasons": candidate.get("target_match_reasons") or [],
|
|
"comparison_mode": candidate.get("target_comparison_mode"),
|
|
"match_type": candidate.get("target_match_type"),
|
|
"hard_veto": bool(candidate.get("target_hard_veto")),
|
|
"target_gap_pct": candidate.get("target_gap_pct"),
|
|
"search_term": candidate.get("target_search_term"),
|
|
"source_contract_guard": guard,
|
|
"source_provenance": provenance,
|
|
"same_item_reconciliation": reconciliation,
|
|
"same_item_candidate_fingerprint": candidate.get(
|
|
"same_item_candidate_fingerprint"
|
|
),
|
|
"tags": [
|
|
"identity_v2",
|
|
f"source_{connector_key}",
|
|
"public_structured_data",
|
|
"product_jsonld_verified",
|
|
"price_basis_total_price",
|
|
"auto_compare_total_price",
|
|
],
|
|
}
|
|
return {
|
|
"source_code": source_code,
|
|
"platform_code": platform_code,
|
|
"source_product_id": source_product_id,
|
|
"source_offer_key": (
|
|
f"{source_code}:{source_product_id}:{pchome_product_id}:total_price"
|
|
),
|
|
"title": title,
|
|
"brand": candidate.get("brand"),
|
|
"category_text": candidate.get("category") or candidate.get("category_text"),
|
|
"product_url": candidate.get("product_url") or candidate.get("url"),
|
|
"image_url": candidate.get("image_url"),
|
|
"price": source_price,
|
|
"original_price": _to_float(candidate.get("original_price")),
|
|
"currency": currency,
|
|
"stock_status": candidate.get("stock_status") or candidate.get("availability"),
|
|
"sold_count": candidate.get("sold_count"),
|
|
"rating": _to_float(candidate.get("rating")),
|
|
"review_count": candidate.get("review_count"),
|
|
"observed_at": observed_at,
|
|
"expires_at": observed_at + timedelta(hours=max(1, min(ttl_hours, 72))),
|
|
"ingestion_method": ingestion_method,
|
|
"connector_key": connector_key,
|
|
"pchome_product_id": pchome_product_id,
|
|
"momo_sku": None,
|
|
"match_status": "verified",
|
|
"quality_score": round(quality_score, 2),
|
|
"data_quality_status": "verified",
|
|
"quality_notes_json": json.dumps(
|
|
[
|
|
f"{source_label}公開結構化商品頁已驗證",
|
|
"PChome 公開規格與獨立同款 verifier 已通過",
|
|
"來源 canary 啟用前已完成 exact DB readback",
|
|
],
|
|
ensure_ascii=False,
|
|
),
|
|
"raw_payload_json": json.dumps(raw_payload, ensure_ascii=False),
|
|
}, ""
|
|
|
|
|
|
def sync_targeted_marketplace_candidates_to_external_offers(
|
|
engine,
|
|
candidates: list[dict[str, Any]],
|
|
*,
|
|
source_code: str,
|
|
platform_code: str,
|
|
ingestion_method: str,
|
|
connector_key: str,
|
|
source_label: str,
|
|
dry_run: bool = False,
|
|
ttl_hours: int = 30,
|
|
) -> dict[str, Any]:
|
|
"""Controlled apply for exact, independently verified marketplace offers."""
|
|
generated_at = datetime.now().isoformat(timespec="seconds")
|
|
candidates = list(candidates or [])
|
|
required_tables = {"external_market_sources", "external_offers"}
|
|
|
|
with engine.begin() as conn:
|
|
missing_tables = sorted(
|
|
table for table in required_tables if not _has_table(conn, table)
|
|
)
|
|
if missing_tables:
|
|
return {
|
|
"success": False,
|
|
"status": "skipped",
|
|
"generated_at": generated_at,
|
|
"candidate_count": len(candidates),
|
|
"written_count": 0,
|
|
"dry_run": dry_run,
|
|
"missing_tables": missing_tables,
|
|
"message": "外部報價同步暫時無法執行，缺少必要資料表。",
|
|
}
|
|
|
|
if not dry_run:
|
|
_ensure_external_market_source_seeds(conn)
|
|
source_exists = conn.execute(
|
|
text(
|
|
"""
|
|
SELECT 1
|
|
FROM external_market_sources
|
|
WHERE code = :source_code
|
|
"""
|
|
),
|
|
{"source_code": source_code},
|
|
).first()
|
|
if not source_exists:
|
|
return {
|
|
"success": False,
|
|
"status": "unregistered_source",
|
|
"generated_at": generated_at,
|
|
"candidate_count": len(candidates),
|
|
"written_count": 0,
|
|
"dry_run": False,
|
|
"source_code": source_code,
|
|
"message": "外部來源尚未完成 registry 登記，本輪沒有寫入。",
|
|
}
|
|
base_observed_at = datetime.now(timezone.utc)
|
|
ranked_offers: list[
|
|
tuple[dict[str, Any], tuple[float, float, float, float]]
|
|
] = []
|
|
skipped_reasons: dict[str, int] = {}
|
|
for index, candidate in enumerate(candidates):
|
|
offer, reason = _targeted_marketplace_candidate_to_external_offer(
|
|
candidate,
|
|
source_code=source_code,
|
|
platform_code=platform_code,
|
|
ingestion_method=ingestion_method,
|
|
connector_key=connector_key,
|
|
source_label=source_label,
|
|
observed_at=base_observed_at + timedelta(microseconds=index),
|
|
ttl_hours=ttl_hours,
|
|
)
|
|
if offer:
|
|
ranked_offers.append((offer, _targeted_candidate_sync_rank(candidate)))
|
|
else:
|
|
skipped_reasons[reason] = skipped_reasons.get(reason, 0) + 1
|
|
|
|
selected_by_pchome: dict[
|
|
str,
|
|
tuple[dict[str, Any], tuple[float, float, float, float]],
|
|
] = {}
|
|
for offer, rank in ranked_offers:
|
|
key = str(offer.get("pchome_product_id") or "").strip()
|
|
existing = selected_by_pchome.get(key)
|
|
if existing is None or rank > existing[1]:
|
|
selected_by_pchome[key] = (offer, rank)
|
|
offers = [offer for offer, _ in selected_by_pchome.values()]
|
|
|
|
if not dry_run:
|
|
for offer in offers:
|
|
_upsert_external_offer(conn, offer)
|
|
|
|
if offers and not dry_run:
|
|
mark_pchome_growth_cache_stale()
|
|
|
|
return {
|
|
"success": True,
|
|
"status": "dry_run" if dry_run else "synced",
|
|
"generated_at": generated_at,
|
|
"candidate_count": len(candidates),
|
|
"selected_count": len(offers),
|
|
"written_count": 0 if dry_run else len(offers),
|
|
"dry_run": dry_run,
|
|
"source_code": source_code,
|
|
"platform_code": platform_code,
|
|
"ingestion_method": ingestion_method,
|
|
"skipped_reasons": skipped_reasons,
|
|
"writes_database_count": 0 if dry_run else len(offers),
|
|
"message": (
|
|
f"已把 {source_label} exact 同款寫入外部報價並等待來源 canary。"
|
|
if not dry_run
|
|
else f"已完成 {source_label} exact 同款寫入預檢，尚未寫入資料。"
|
|
),
|
|
}
|
|
|
|
|
|
def activate_external_market_source_after_canary(
|
|
engine,
|
|
*,
|
|
source_code: str,
|
|
candidates: list[dict[str, Any]],
|
|
independent_readback: dict[str, Any],
|
|
identity: dict[str, str],
|
|
minimum_verified_count: int = 2,
|
|
) -> dict[str, Any]:
|
|
"""Activate a source only after a bounded exact-write canary is read back."""
|
|
candidates = list(candidates or [])
|
|
threshold = max(2, min(int(minimum_verified_count or 2), 20))
|
|
readback_count = int(independent_readback.get("readback_pass_count") or 0)
|
|
guards_passed = bool(candidates) and all(
|
|
isinstance(item.get("source_contract_guard"), dict)
|
|
and bool(item["source_contract_guard"].get("passed"))
|
|
for item in candidates
|
|
)
|
|
independent_verifiers_passed = bool(candidates) and all(
|
|
isinstance(item.get("same_item_reconciliation"), dict)
|
|
and bool(item["same_item_reconciliation"].get("independent_verifier_passed"))
|
|
for item in candidates
|
|
)
|
|
canary_evidence_verified = bool(
|
|
independent_readback.get("success")
|
|
and readback_count == len(candidates)
|
|
and guards_passed
|
|
and independent_verifiers_passed
|
|
)
|
|
if not canary_evidence_verified:
|
|
return {
|
|
"success": True,
|
|
"status": "canary_threshold_not_met",
|
|
"source_code": source_code,
|
|
"candidate_count": len(candidates),
|
|
"readback_pass_count": readback_count,
|
|
"minimum_verified_count": threshold,
|
|
"guards_passed": guards_passed,
|
|
"independent_verifiers_passed": independent_verifiers_passed,
|
|
"applied": False,
|
|
"state_changed": False,
|
|
"writes_database_count": 0,
|
|
}
|
|
|
|
with engine.begin() as conn:
|
|
_ensure_external_market_source_seeds(conn)
|
|
current = (
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
SELECT status, enabled, write_enabled
|
|
FROM external_market_sources
|
|
WHERE code = :source_code
|
|
"""
|
|
),
|
|
{"source_code": source_code},
|
|
)
|
|
.mappings()
|
|
.first()
|
|
)
|
|
if not current:
|
|
return {
|
|
"success": False,
|
|
"status": "source_registry_missing",
|
|
"source_code": source_code,
|
|
"applied": False,
|
|
"state_changed": False,
|
|
"writes_database_count": 0,
|
|
}
|
|
|
|
already_active = bool(
|
|
str(current.get("status") or "") == "active"
|
|
and bool(current.get("enabled"))
|
|
and bool(current.get("write_enabled"))
|
|
)
|
|
if already_active:
|
|
return {
|
|
"success": True,
|
|
"status": "already_active_verified",
|
|
"source_code": source_code,
|
|
"candidate_count": len(candidates),
|
|
"readback_pass_count": readback_count,
|
|
"minimum_verified_count": threshold,
|
|
"guards_passed": guards_passed,
|
|
"independent_verifiers_passed": independent_verifiers_passed,
|
|
"applied": True,
|
|
"state_changed": False,
|
|
"previous_state": dict(current),
|
|
"observed_state": dict(current),
|
|
"identity": dict(identity or {}),
|
|
"writes_database_count": 0,
|
|
}
|
|
|
|
if len(candidates) < threshold:
|
|
return {
|
|
"success": True,
|
|
"status": "canary_threshold_not_met",
|
|
"source_code": source_code,
|
|
"candidate_count": len(candidates),
|
|
"readback_pass_count": readback_count,
|
|
"minimum_verified_count": threshold,
|
|
"guards_passed": guards_passed,
|
|
"independent_verifiers_passed": independent_verifiers_passed,
|
|
"applied": False,
|
|
"state_changed": False,
|
|
"observed_state": dict(current),
|
|
"writes_database_count": 0,
|
|
}
|
|
|
|
with engine.begin() as conn:
|
|
_ensure_external_market_source_seeds(conn)
|
|
previous = (
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
SELECT status, enabled, write_enabled
|
|
FROM external_market_sources
|
|
WHERE code = :source_code
|
|
"""
|
|
),
|
|
{"source_code": source_code},
|
|
)
|
|
.mappings()
|
|
.first()
|
|
)
|
|
if not previous:
|
|
return {
|
|
"success": False,
|
|
"status": "source_registry_missing",
|
|
"source_code": source_code,
|
|
"applied": False,
|
|
"state_changed": False,
|
|
"writes_database_count": 0,
|
|
}
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
UPDATE external_market_sources
|
|
SET status = 'active',
|
|
enabled = :enabled,
|
|
write_enabled = :write_enabled,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE code = :source_code
|
|
"""
|
|
),
|
|
{"source_code": source_code, "enabled": True, "write_enabled": True},
|
|
)
|
|
observed = (
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
SELECT status, enabled, write_enabled
|
|
FROM external_market_sources
|
|
WHERE code = :source_code
|
|
"""
|
|
),
|
|
{"source_code": source_code},
|
|
)
|
|
.mappings()
|
|
.first()
|
|
)
|
|
|
|
applied = bool(
|
|
observed
|
|
and str(observed.get("status") or "") == "active"
|
|
and bool(observed.get("enabled"))
|
|
and bool(observed.get("write_enabled"))
|
|
)
|
|
if applied:
|
|
mark_pchome_growth_cache_stale()
|
|
return {
|
|
"success": applied,
|
|
"status": "activated" if applied else "activation_readback_failed",
|
|
"source_code": source_code,
|
|
"candidate_count": len(candidates),
|
|
"readback_pass_count": readback_count,
|
|
"minimum_verified_count": threshold,
|
|
"guards_passed": guards_passed,
|
|
"independent_verifiers_passed": independent_verifiers_passed,
|
|
"applied": applied,
|
|
"previous_state": dict(previous or {}),
|
|
"observed_state": dict(observed or {}),
|
|
"identity": dict(identity or {}),
|
|
"writes_database_count": 1 if applied else 0,
|
|
"state_changed": applied,
|
|
}
|
|
|
|
|
|
def rollback_external_market_source_canary(
|
|
engine,
|
|
*,
|
|
source_code: str,
|
|
ingestion_method: str,
|
|
candidates: list[dict[str, Any]],
|
|
identity: dict[str, str],
|
|
activation_receipt: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
"""Restore source state and quarantine only rows written by the failed run."""
|
|
previous = activation_receipt.get("previous_state")
|
|
previous = previous if isinstance(previous, dict) else {}
|
|
run_id = str((identity or {}).get("run_id") or "")
|
|
quarantined_ids: list[int] = []
|
|
|
|
with engine.begin() as conn:
|
|
for candidate in candidates or []:
|
|
rows = (
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
SELECT id, raw_payload_json
|
|
FROM external_offers
|
|
WHERE source_code = :source_code
|
|
AND ingestion_method = :ingestion_method
|
|
AND pchome_product_id = :pchome_product_id
|
|
AND source_product_id = :source_product_id
|
|
ORDER BY observed_at DESC, id DESC
|
|
"""
|
|
),
|
|
{
|
|
"source_code": source_code,
|
|
"ingestion_method": ingestion_method,
|
|
"pchome_product_id": candidate.get("target_pchome_product_id"),
|
|
"source_product_id": (
|
|
candidate.get("product_id")
|
|
or candidate.get("goodsCode")
|
|
or candidate.get("id")
|
|
),
|
|
},
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
for row in rows:
|
|
raw = _load_json_dict(row.get("raw_payload_json"))
|
|
observed_run_id = str(
|
|
(raw.get("same_item_reconciliation") or {}).get("run_id") or ""
|
|
)
|
|
if run_id and observed_run_id == run_id:
|
|
quarantined_ids.append(int(row["id"]))
|
|
|
|
quarantined_ids = sorted(set(quarantined_ids))
|
|
for offer_id in quarantined_ids:
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
UPDATE external_offers
|
|
SET match_status = 'quarantined',
|
|
data_quality_status = 'needs_review',
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = :offer_id
|
|
"""
|
|
),
|
|
{"offer_id": offer_id},
|
|
)
|
|
|
|
state_changed = bool(
|
|
activation_receipt.get(
|
|
"state_changed",
|
|
activation_receipt.get("applied"),
|
|
)
|
|
)
|
|
if state_changed and previous:
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
UPDATE external_market_sources
|
|
SET status = :status,
|
|
enabled = :enabled,
|
|
write_enabled = :write_enabled,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE code = :source_code
|
|
"""
|
|
),
|
|
{
|
|
"source_code": source_code,
|
|
"status": previous.get("status") or "paused",
|
|
"enabled": bool(previous.get("enabled")),
|
|
"write_enabled": bool(previous.get("write_enabled")),
|
|
},
|
|
)
|
|
|
|
mark_pchome_growth_cache_stale()
|
|
return {
|
|
"success": True,
|
|
"status": "rolled_back_and_quarantined",
|
|
"source_code": source_code,
|
|
"quarantined_offer_count": len(quarantined_ids),
|
|
"source_state_restored": bool(state_changed and previous),
|
|
"identity": dict(identity or {}),
|
|
"writes_database_count": len(quarantined_ids)
|
|
+ (1 if state_changed and previous else 0),
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"activate_external_market_source_after_canary",
|
|
"rollback_external_market_source_canary",
|
|
"sync_targeted_marketplace_candidates_to_external_offers",
|
|
]
|