"""市場情報專業來源治理 gate。 本模組把主流市場資料採集做法轉成可審核合約： robots/REP、sitemap、structured data、canonical URL、rate limit、 公開資料邊界、provenance、snapshot hash 與 idempotency。 API/UI 只審核操作員提供的治理摘要；不抓外站、不讀 robots/sitemap、 不開 DB、不寫檔、不掛 scheduler。 """ from urllib.parse import urlparse from services.market_intel.mcp_fetch_candidate_queue_writer_post_closeout_inventory_review import ( FORBIDDEN_SECRET_KEYS, SAFE_SECRET_METADATA_KEYS, _as_dict, _as_list, _blocked_side_effects, _contains_forbidden_key, _safe_int, _safe_path, _safe_text, ) from services.market_intel.mcp_fetch_candidate_queue_writer_run_readiness import ( ARTIFACT_PREFIX, ) from services.market_intel.mcp_professional_source_governance_gates import ( CONTRACT_SCOPE, SOURCE_POLICY_VERSION, _is_public_http_url, build_professional_source_governance_gates, ) from services.market_intel.mcp_professional_source_governance_sample import ( build_sample_professional_source_governance_package, ) _BLOCKED_SOURCE_GOVERNANCE_SIDE_EFFECT_KEYS = ( "allow_api_database_write", "allow_api_execution", "allow_api_file_write", "allow_api_network_fetch", "allow_database_write", "allow_external_network", "allow_scheduler_attach", "api_fetches_robots_txt", "api_fetches_sitemap", "api_fetches_source_url", "api_opens_database_connection", "api_uses_external_network", "api_writes_database", "api_writes_file", "database_commit_executed", "database_write_executed", "external_network_executed", "fetch_executed", "file_written", "network_request_allowed", "payload_persisted", "ready_for_api_database_write", "real_write_allowed_by_api", "scheduler_attached", "write_database", "writes_executed", "would_write_database", ) _RAW_PAYLOAD_KEYS = ( "body_html", "full_response_body", "html", "page_body", "page_html", "raw_html", "raw_page_html", "response_body", ) def _safe_float(value): try: return float(value or 0) except (TypeError, ValueError): return 0.0 def _contains_raw_payload(value): if isinstance(value, dict): for key, nested in value.items(): if str(key).lower() in _RAW_PAYLOAD_KEYS and bool(nested): return True if _contains_raw_payload(nested): return True if isinstance(value, list): return any(_contains_raw_payload(item) for item in value) return False def _blocked_source_governance_side_effects(payload): found = list(_blocked_side_effects(payload)) def visit(value, path): if isinstance(value, dict): for key, item in value.items(): normalized_key = str(key).lower() key_path = f"{path}.{key}" if path else key if ( normalized_key in _BLOCKED_SOURCE_GOVERNANCE_SIDE_EFFECT_KEYS and bool(item) ): found.append(key_path) visit(item, key_path) elif isinstance(value, list): for index, item in enumerate(value): visit(item, f"{path}[{index}]") visit(payload, "") return sorted(set(found)) def _normalize_host(value): if not value: return None parsed = urlparse(value) return parsed.netloc.lower() or None def _source_summary(source): source = _as_dict(source) source_url = _safe_text(source.get("source_url"), 500) canonical_url = _safe_text(source.get("canonical_url"), 500) robots_url = _safe_text(source.get("robots_url"), 500) sitemap_url = _safe_text(source.get("sitemap_url"), 500) structured_data_types = [ _safe_text(item, 80) for item in _as_list(source.get("structured_data_types")) if _safe_text(item, 80) ] max_requests = _safe_int(source.get("max_requests_per_run")) crawl_delay_seconds = _safe_float(source.get("crawl_delay_seconds")) evidence_artifact_path = _safe_text(source.get("evidence_artifact_path")) source_host = _normalize_host(source_url) canonical_host = _normalize_host(canonical_url) return { "platform_code": _safe_text(source.get("platform_code"), 80), "source_key": _safe_text(source.get("source_key"), 160), "source_url": source_url, "canonical_url": canonical_url, "robots_url": robots_url, "sitemap_url": sitemap_url, "lastmod_source": _safe_text(source.get("lastmod_source"), 160), "source_url_safe": _is_public_http_url(source_url), "canonical_url_safe": _is_public_http_url(canonical_url), "robots_url_safe": _is_public_http_url(robots_url), "sitemap_url_safe": _is_public_http_url(sitemap_url), "canonical_host_matches_source": bool( source_host and canonical_host and source_host == canonical_host ), "robots_policy_checked": bool(source.get("robots_policy_checked")), "robots_allowed": bool(source.get("robots_allowed")), "tos_public_page_checked": bool(source.get("tos_public_page_checked")), "login_required": bool(source.get("login_required")), "member_or_order_data": bool(source.get("member_or_order_data")), "cart_order_or_pii": bool(source.get("cart_order_or_pii")), "anti_bot_bypass_required": bool(source.get("anti_bot_bypass_required")), "structured_data_preferred": bool(source.get("structured_data_preferred")), "json_ld_first": bool(source.get("json_ld_first")), "dom_selector_fallback_allowed": bool( source.get("dom_selector_fallback_allowed") ), "structured_data_types": structured_data_types, "selector_version": _safe_text(source.get("selector_version"), 120), "crawl_delay_seconds": crawl_delay_seconds, "max_requests_per_run": max_requests, "public_cache_ttl_hours": _safe_int(source.get("public_cache_ttl_hours")), "evidence_artifact_path": evidence_artifact_path, "evidence_artifact_path_safe": _safe_path( evidence_artifact_path, prefixes=(ARTIFACT_PREFIX,), suffixes=(".json",), ), "provenance_required": bool(source.get("provenance_required")), "snapshot_hash_required": bool(source.get("snapshot_hash_required")), "idempotency_key_strategy": _safe_text( source.get("idempotency_key_strategy"), 160 ), } def _operator_confirmations(payload): confirmations = _as_dict(payload.get("operator_confirmations")) return { "human_reviewed_source_policy": bool( confirmations.get("human_reviewed_source_policy") ), "robots_and_tos_checked_by_operator": bool( confirmations.get("robots_and_tos_checked_by_operator") ), "public_pages_only": bool(confirmations.get("public_pages_only")), "no_login_or_member_data": bool( confirmations.get("no_login_or_member_data") ), "no_cart_order_or_pii": bool(confirmations.get("no_cart_order_or_pii")), "no_antibot_bypass": bool(confirmations.get("no_antibot_bypass")), "structured_data_first": bool(confirmations.get("structured_data_first")), "provenance_required": bool(confirmations.get("provenance_required")), "no_api_network_fetch": bool(confirmations.get("no_api_network_fetch")), "no_database_write": bool(confirmations.get("no_database_write")), "no_scheduler_attach": bool(confirmations.get("no_scheduler_attach")), "no_secret_payload": bool(confirmations.get("no_secret_payload")), } def _governance_summary(payload): payload = _as_dict(payload) sources = [_source_summary(source) for source in _as_list(payload.get("sources"))] blocked_side_effects = _blocked_source_governance_side_effects(payload) raw_payload_submitted = _contains_raw_payload(payload) secret_or_token_submitted = _contains_forbidden_key( payload, FORBIDDEN_SECRET_KEYS, safe_keys=SAFE_SECRET_METADATA_KEYS | { "no_api_network_fetch", "no_secret_payload", "source_contract_version", }, ) return { "governance_id": _safe_text(payload.get("governance_id"), 160), "governance_scope": _safe_text(payload.get("governance_scope"), 160), "policy_version": _safe_text(payload.get("policy_version"), 120), "source_contract_version": _safe_text( payload.get("source_contract_version"), 120 ), "sources": sources, "source_count": len(sources), "platform_count": len( {source["platform_code"] for source in sources if source["platform_code"]} ), "robots_checked_count": len( [ source for source in sources if source["robots_policy_checked"] and source["robots_allowed"] ] ), "structured_data_ready_count": len( [ source for source in sources if source["structured_data_preferred"] and source["json_ld_first"] ] ), "min_crawl_delay_seconds": min( [source["crawl_delay_seconds"] for source in sources] or [0.0] ), "max_requests_per_run": max( [source["max_requests_per_run"] for source in sources] or [0] ), "operator_confirmations": _operator_confirmations(payload), "raw_payload_submitted_to_api": raw_payload_submitted, "secret_or_token_submitted_to_api": secret_or_token_submitted, "blocked_side_effects": blocked_side_effects, "api_uses_external_network": bool(payload.get("api_uses_external_network")), "api_fetches_robots_txt": bool(payload.get("api_fetches_robots_txt")), "api_fetches_sitemap": bool(payload.get("api_fetches_sitemap")), "api_fetches_source_url": bool(payload.get("api_fetches_source_url")), "api_opens_database_connection": bool( payload.get("api_opens_database_connection") ), "api_writes_database": bool(payload.get("api_writes_database")), "api_writes_file": bool(payload.get("api_writes_file")), "scheduler_attached": bool(payload.get("scheduler_attached")), } def _source_contract(): return { "contract_scope": CONTRACT_SCOPE, "policy_version": SOURCE_POLICY_VERSION, "mainstream_practices": [ { "key": "robots_exclusion_protocol", "label": "先人工確認 robots.txt / REP，不由 API 自動抓取或繞過", "reference_url": "https://www.rfc-editor.org/rfc/rfc9309", }, { "key": "sitemap_provenance", "label": "以 sitemap / lastmod 作為活動來源發現與更新依據之一", "reference_url": "https://www.sitemaps.org/protocol.html", }, { "key": "structured_data_first", "label": "優先解析 JSON-LD / schema.org Product、Offer、ItemList", "reference_url": "https://developers.google.com/search/docs/appearance/structured-data/product-snippet", }, { "key": "canonical_public_url", "label": "所有來源需保留 canonical URL、公開 URL 與 host provenance", "reference_url": "https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls", }, { "key": "bronze_silver_gold", "label": "raw evidence、normalized source、reviewed product/match 分層保存", "reference_url": "internal:market_intel_lakehouse_contract", }, ], "required_source_fields": [ "platform_code", "source_key", "source_url", "canonical_url", "robots_url", "sitemap_url", "robots_policy_checked", "robots_allowed", "structured_data_types", "crawl_delay_seconds", "max_requests_per_run", "evidence_artifact_path", "snapshot_hash_required", "idempotency_key_strategy", ], "forbidden_data": [ "login_page", "member_profile", "cart_or_checkout", "order_data", "personal_data", "cookie_or_token", "anti_bot_bypass", ], "next_gate": "mcp_fetch_target_review_with_source_governance", } def build_mcp_professional_source_governance_preview( *, operator_source_governance=None, phase=None ): payload_received = operator_source_governance is not None governance_payload = _as_dict(operator_source_governance) governance = _governance_summary(governance_payload) gates = build_professional_source_governance_gates( package_received=payload_received, governance=governance, ) blocked_reasons = [gate["key"] for gate in gates if not gate["passed"]] accepted = bool(payload_received and not blocked_reasons) return { "mode": ( "mcp_professional_source_governance" if accepted else "mcp_professional_source_governance_preview" ), "phase": phase, "source_governance_payload_received": payload_received, "mcp_professional_source_governance_accepted": accepted, "ready_for_mcp_fetch_source_contract": accepted, "ready_for_api_database_write": False, "ready_for_scheduler_attach": False, "network_request_allowed": False, "external_network_executed": False, "api_uses_external_network": False, "api_fetches_robots_txt": False, "api_fetches_sitemap": False, "api_fetches_source_url": False, "api_opens_database_connection": False, "api_writes_database": False, "api_writes_file": False, "database_connection_opened": False, "database_write_executed": False, "database_commit_executed": False, "file_written": False, "payload_persisted": False, "scheduler_attached": False, "gate_count": len(gates), "passed_gate_count": len([gate for gate in gates if gate["passed"]]), "blocked_reasons": blocked_reasons, "gates": gates, "source_governance_summary": governance, "source_contract": _source_contract(), "sources": governance["sources"], "next_operator_steps": [ "人工保留 robots / sitemap / ToS / public URL 審核證據。", "將通過治理的來源餵給後續 fetch target review，不由 API 直接抓外站。", "正式 fetch 前仍需 MCP readiness、外部 MCP health、manual run package 與 receipt gate。", ], "sample_professional_source_governance_package": ( build_sample_professional_source_governance_package() ), }