389 lines
15 KiB
Python
389 lines
15 KiB
Python
"""Read-only evidence enrichment and source-wiring previews."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from services.ai_exception_contract import LEGACY_HUMAN_REVIEW_REQUIRED_COUNT_KEY
|
|
from services.pchome_mapping_backlog.contracts import (
|
|
_ai_exception_compatibility_fields,
|
|
_evidence_requires_ai_exception,
|
|
)
|
|
from services.pchome_mapping_backlog.policies import (
|
|
EVIDENCE_ENRICHMENT_PREVIEW_POLICY,
|
|
EVIDENCE_SOURCE_PREVIEW_POLICY,
|
|
PCHOME_FETCH_MAX_BATCH_SIZE,
|
|
PRODUCT_PAGE_EVIDENCE_PARSER_POLICY,
|
|
)
|
|
from services.pchome_mapping_backlog.reporter import (
|
|
build_pchome_mapping_operator_preview,
|
|
)
|
|
|
|
|
|
def _field_enrichment_sources(field: str) -> list[dict[str, Any]]:
|
|
source_map = {
|
|
"image": [
|
|
{
|
|
"source": "PChome product page structured data",
|
|
"mode": "future_read_only_fetch",
|
|
"writes_database": False,
|
|
"expected_output": "primary product image URL",
|
|
},
|
|
{
|
|
"source": "existing marketplace catalog payload",
|
|
"mode": "reuse_if_present",
|
|
"writes_database": False,
|
|
"expected_output": "cached image_url",
|
|
},
|
|
],
|
|
"availability": [
|
|
{
|
|
"source": "PChome product page offer availability",
|
|
"mode": "future_read_only_fetch",
|
|
"writes_database": False,
|
|
"expected_output": "in_stock / out_of_stock / unknown",
|
|
},
|
|
{
|
|
"source": "merchant listing structured data",
|
|
"mode": "future_read_only_parse",
|
|
"writes_database": False,
|
|
"expected_output": "schema.org Offer availability",
|
|
},
|
|
],
|
|
"unit_price_or_package_basis": [
|
|
{
|
|
"source": "deterministic product title parser",
|
|
"mode": "local_parse_preview",
|
|
"writes_database": False,
|
|
"expected_output": "size, count, unit label, package basis",
|
|
},
|
|
{
|
|
"source": "external_price.price_basis",
|
|
"mode": "reuse_if_present",
|
|
"writes_database": False,
|
|
"expected_output": "total_price / unit_price",
|
|
},
|
|
],
|
|
"price": [
|
|
{
|
|
"source": "growth opportunity payload",
|
|
"mode": "reuse_if_present",
|
|
"writes_database": False,
|
|
"expected_output": "PChome listed price",
|
|
},
|
|
],
|
|
}
|
|
return source_map.get(field, [])
|
|
|
|
|
|
def _source_plan_for_field(field: str, missing_count: int) -> dict[str, Any]:
|
|
plans = {
|
|
"image": {
|
|
"payload_keys_checked": ["image_url", "image", "product_image_url"],
|
|
"preferred_current_source": "existing marketplace catalog payload",
|
|
"future_read_only_fetch_gate": {
|
|
"method": "GET",
|
|
"allowed_domain": "24h.pchome.com.tw",
|
|
"product_url_required": True,
|
|
"parse_targets": [
|
|
"schema.org Product.image",
|
|
"og:image",
|
|
"primary product image",
|
|
],
|
|
"check_mode_parser": PRODUCT_PAGE_EVIDENCE_PARSER_POLICY,
|
|
"fetches_external_sites_in_preview": False,
|
|
"writes_database": False,
|
|
},
|
|
},
|
|
"availability": {
|
|
"payload_keys_checked": [
|
|
"availability",
|
|
"stock_status",
|
|
"is_available",
|
|
],
|
|
"preferred_current_source": "existing marketplace catalog payload",
|
|
"future_read_only_fetch_gate": {
|
|
"method": "GET",
|
|
"allowed_domain": "24h.pchome.com.tw",
|
|
"product_url_required": True,
|
|
"parse_targets": [
|
|
"schema.org Offer.availability",
|
|
"merchant listing offer availability",
|
|
],
|
|
"check_mode_parser": PRODUCT_PAGE_EVIDENCE_PARSER_POLICY,
|
|
"fetches_external_sites_in_preview": False,
|
|
"writes_database": False,
|
|
},
|
|
},
|
|
"price": {
|
|
"payload_keys_checked": [
|
|
"pchome_price",
|
|
"external_price.pchome_price",
|
|
"review_candidate.pchome_price",
|
|
],
|
|
"preferred_current_source": "growth opportunity payload",
|
|
"future_read_only_fetch_gate": None,
|
|
"payload_mapping_probe": {
|
|
"goal": "Confirm whether missing price is a source payload gap or summary field mapping gap.",
|
|
"fetches_external_sites_in_preview": False,
|
|
"writes_database": False,
|
|
},
|
|
},
|
|
}
|
|
plan = dict(plans.get(field, {}))
|
|
if not plan:
|
|
return {}
|
|
plan["field"] = field
|
|
plan["status"] = (
|
|
"missing_in_current_payload"
|
|
if missing_count
|
|
else "covered_by_current_payload"
|
|
)
|
|
plan["missing_count"] = missing_count
|
|
return plan
|
|
|
|
|
|
def _build_fetch_gate_candidates(
|
|
tasks: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
candidates = []
|
|
for task in tasks:
|
|
missing_fields = set(task.get("missing_fields") or [])
|
|
fetch_fields = [
|
|
field
|
|
for field in ("image", "availability")
|
|
if field in missing_fields
|
|
]
|
|
if not fetch_fields:
|
|
continue
|
|
candidates.append(
|
|
{
|
|
"pchome_product_id": task.get("pchome_product_id") or "",
|
|
"product_name": task.get("product_name") or "",
|
|
"product_url": task.get("product_url"),
|
|
"fields": fetch_fields,
|
|
"method": "GET",
|
|
"allowed_domain": "24h.pchome.com.tw",
|
|
"executes_fetch_in_preview": False,
|
|
"writes_database": False,
|
|
}
|
|
)
|
|
return candidates[:PCHOME_FETCH_MAX_BATCH_SIZE]
|
|
|
|
|
|
def _build_evidence_task(target: dict[str, Any], lane: str) -> dict[str, Any]:
|
|
evidence = target.get("evidence_completeness") or {}
|
|
missing_fields = list(evidence.get("missing_fields") or [])
|
|
blocking_missing_fields = list(evidence.get("blocking_missing_fields") or [])
|
|
enrichment_steps = [
|
|
{
|
|
"field": field,
|
|
"blocking": field in blocking_missing_fields,
|
|
"sources": _field_enrichment_sources(field),
|
|
}
|
|
for field in missing_fields
|
|
]
|
|
ai_exception_required = _evidence_requires_ai_exception(evidence)
|
|
|
|
return {
|
|
"lane": lane,
|
|
"pchome_product_id": target.get("pchome_product_id") or "",
|
|
"product_name": target.get("product_name") or "",
|
|
"product_url": target.get("product_url") or evidence.get("product_url"),
|
|
"sales_7d": target.get("sales_7d"),
|
|
"priority_score": target.get("priority_score"),
|
|
"action_code": target.get("action_code"),
|
|
"evidence_score": evidence.get("score"),
|
|
"present_fields": list(evidence.get("present_fields") or []),
|
|
"missing_fields": missing_fields,
|
|
"blocking_missing_fields": blocking_missing_fields,
|
|
"auto_accept_ready": bool(evidence.get("auto_accept_ready")),
|
|
**_ai_exception_compatibility_fields(ai_exception_required),
|
|
"unit_package_basis": evidence.get("unit_package_basis"),
|
|
"enrichment_steps": enrichment_steps,
|
|
}
|
|
|
|
|
|
def build_pchome_evidence_enrichment_preview(
|
|
payload: dict[str, Any],
|
|
batch_size: int = 5,
|
|
) -> dict[str, Any]:
|
|
"""Build a read-only evidence enrichment package for mapping targets."""
|
|
operator_preview = build_pchome_mapping_operator_preview(
|
|
payload,
|
|
batch_size=batch_size,
|
|
)
|
|
operator_batch = operator_preview.get("operator_batch") or {}
|
|
direct_targets = list(operator_batch.get("direct_mapping_targets") or [])
|
|
review_targets = list(operator_batch.get("review_candidate_targets") or [])
|
|
evidence_tasks = [
|
|
*[
|
|
_build_evidence_task(target, "direct_mapping")
|
|
for target in direct_targets
|
|
],
|
|
*[
|
|
_build_evidence_task(target, "review_candidate")
|
|
for target in review_targets
|
|
],
|
|
]
|
|
tasks_with_blockers = [
|
|
task for task in evidence_tasks if task.get("blocking_missing_fields")
|
|
]
|
|
missing_field_counts: dict[str, int] = {}
|
|
for task in evidence_tasks:
|
|
for field in task.get("missing_fields") or []:
|
|
missing_field_counts[field] = missing_field_counts.get(field, 0) + 1
|
|
|
|
if tasks_with_blockers:
|
|
result = "NEEDS_EVIDENCE_ENRICHMENT"
|
|
elif evidence_tasks:
|
|
result = "EVIDENCE_PREVIEW_READY"
|
|
else:
|
|
result = "NO_TARGETS"
|
|
|
|
return {
|
|
"policy": EVIDENCE_ENRICHMENT_PREVIEW_POLICY,
|
|
"result": result,
|
|
"success": bool(operator_preview.get("success")),
|
|
"generated_at": operator_preview.get("generated_at"),
|
|
"source_policy": operator_preview.get("policy"),
|
|
"stats": operator_preview.get("stats") or {},
|
|
"summary": {
|
|
"task_count": len(evidence_tasks),
|
|
"tasks_with_blockers": len(tasks_with_blockers),
|
|
"missing_field_counts": missing_field_counts,
|
|
"auto_accept_ready_count": sum(
|
|
1 for task in evidence_tasks if task.get("auto_accept_ready")
|
|
),
|
|
LEGACY_HUMAN_REVIEW_REQUIRED_COUNT_KEY: 0,
|
|
"primary_human_gate_count": 0,
|
|
"ai_exception_required_count": sum(
|
|
1
|
|
for task in evidence_tasks
|
|
if task.get("ai_exception_required")
|
|
),
|
|
},
|
|
"evidence_tasks": evidence_tasks,
|
|
"external_benchmark_alignment": (
|
|
operator_preview.get("external_benchmark_alignment") or {}
|
|
),
|
|
"ai_automation_plan": {
|
|
"policy": "ollama_first_read_only_evidence_assist",
|
|
"llm_calls_in_preview": False,
|
|
"gemini_allowed": False,
|
|
"can_execute_write": False,
|
|
"recommended_next_ai_task": "Generate deterministic identity anchors and evidence gap summaries after image and availability sources are wired.",
|
|
"blocked_until": [
|
|
"image evidence source is wired",
|
|
"availability evidence source is wired",
|
|
"unit/package parser preview is compared against production title samples",
|
|
],
|
|
},
|
|
"safety": {
|
|
"read_only_preview": True,
|
|
"fetches_external_sites": False,
|
|
"writes_database": False,
|
|
"executes_search": False,
|
|
"dispatches_telegram": False,
|
|
"llm_calls_in_preview": False,
|
|
"requires_operator_write_approval": True,
|
|
},
|
|
"next_actions": [
|
|
"Wire read-only image and availability enrichment before expanding auto-accept.",
|
|
"Validate deterministic unit/package basis parsing for bundle-sensitive items.",
|
|
"Keep DB writes behind the existing /api/ai/pchome-growth/backfill-momo-candidates write gate.",
|
|
],
|
|
}
|
|
|
|
|
|
def build_pchome_evidence_source_preview(
|
|
payload: dict[str, Any],
|
|
batch_size: int = 5,
|
|
) -> dict[str, Any]:
|
|
"""Build a read-only source wiring preview for missing product evidence fields."""
|
|
enrichment_preview = build_pchome_evidence_enrichment_preview(
|
|
payload,
|
|
batch_size=batch_size,
|
|
)
|
|
tasks = list(enrichment_preview.get("evidence_tasks") or [])
|
|
fields = ["image", "availability", "price", "unit_price_or_package_basis"]
|
|
field_counts = {}
|
|
source_plans = {}
|
|
for field in fields:
|
|
missing_tasks = [
|
|
task
|
|
for task in tasks
|
|
if field in (task.get("missing_fields") or [])
|
|
]
|
|
field_counts[field] = {
|
|
"missing_count": len(missing_tasks),
|
|
"present_count": max(len(tasks) - len(missing_tasks), 0),
|
|
"sample_missing_targets": [
|
|
{
|
|
"pchome_product_id": task.get("pchome_product_id") or "",
|
|
"product_name": task.get("product_name") or "",
|
|
"product_url": task.get("product_url"),
|
|
"lane": task.get("lane"),
|
|
}
|
|
for task in missing_tasks[:3]
|
|
],
|
|
}
|
|
plan = _source_plan_for_field(field, len(missing_tasks))
|
|
if plan:
|
|
source_plans[field] = plan
|
|
|
|
fetch_gate_candidates = _build_fetch_gate_candidates(tasks)
|
|
if (
|
|
field_counts["image"]["missing_count"]
|
|
or field_counts["availability"]["missing_count"]
|
|
):
|
|
result = "NEEDS_SOURCE_WIRING"
|
|
elif field_counts["price"]["missing_count"]:
|
|
result = "NEEDS_PAYLOAD_MAPPING_PROBE"
|
|
elif tasks:
|
|
result = "SOURCE_PREVIEW_READY"
|
|
else:
|
|
result = "NO_TARGETS"
|
|
|
|
return {
|
|
"policy": EVIDENCE_SOURCE_PREVIEW_POLICY,
|
|
"result": result,
|
|
"success": bool(enrichment_preview.get("success")),
|
|
"generated_at": enrichment_preview.get("generated_at"),
|
|
"source_policy": enrichment_preview.get("policy"),
|
|
"stats": enrichment_preview.get("stats") or {},
|
|
"summary": {
|
|
"task_count": len(tasks),
|
|
"field_counts": field_counts,
|
|
"fetch_gate_candidate_count": len(fetch_gate_candidates),
|
|
},
|
|
"source_plans": source_plans,
|
|
"fetch_gate_candidates": fetch_gate_candidates,
|
|
"external_benchmark_alignment": (
|
|
enrichment_preview.get("external_benchmark_alignment") or {}
|
|
),
|
|
"ai_automation_plan": {
|
|
"policy": "ollama_first_read_only_source_wiring_assist",
|
|
"llm_calls_in_preview": False,
|
|
"gemini_allowed": False,
|
|
"can_execute_fetch": False,
|
|
"can_execute_write": False,
|
|
"recommended_next_ai_task": "Generate schema-aware parsers for image and availability after fetch-gate tests are accepted.",
|
|
},
|
|
"safety": {
|
|
"read_only_preview": True,
|
|
"fetches_external_sites": False,
|
|
"writes_database": False,
|
|
"executes_search": False,
|
|
"dispatches_telegram": False,
|
|
"llm_calls_in_preview": False,
|
|
"requires_production_version_truth": True,
|
|
"requires_fetch_gate_implementation_before_external_get": True,
|
|
},
|
|
"next_actions": [
|
|
"Wire image source preview from existing payload keys before adding a controlled product-page fetch gate.",
|
|
"Wire availability source preview from existing payload keys before adding a controlled Offer availability parser.",
|
|
"Probe missing price rows through current payload mapping before any external fetch or write.",
|
|
],
|
|
}
|