V10.496 add market intel queue writer cli review gate

This commit is contained in:
OoO
2026-05-31 13:52:05 +08:00
parent d1a08b0b37
commit b109bbf3c4
12 changed files with 1242 additions and 179 deletions

View File

@@ -84,6 +84,9 @@ from services.market_intel.mcp_fetch_candidate_queue_review import (
from services.market_intel.mcp_fetch_candidate_queue_writer_preflight import (
build_mcp_fetch_candidate_queue_writer_preflight_preview,
)
from services.market_intel.mcp_fetch_candidate_queue_writer_cli_review import (
build_mcp_fetch_candidate_queue_writer_cli_review_preview,
)
from services.market_intel.mcp_manual_fetch_handoff import (
build_mcp_manual_fetch_handoff_preview,
)
@@ -221,6 +224,11 @@ PRODUCTION_SMOKE_TARGETS = (
+ ("/api/market_intel/mcp_fetch_candidate_queue_writer_preflight",)
+ PRODUCTION_SMOKE_TARGETS[-1:]
)
PRODUCTION_SMOKE_TARGETS = (
PRODUCTION_SMOKE_TARGETS[:-1]
+ ("/api/market_intel/mcp_fetch_candidate_queue_writer_cli_review",)
+ PRODUCTION_SMOKE_TARGETS[-1:]
)
def _run_review_preview_safe(payload, mode):
return bool(payload["mode"] == mode and all(not payload.get(key) for key in BLOCKED_RUN_REVIEW_KEYS))
def build_deployment_readiness_preview(*, service, market_intel_tables, schema_smoke_builder):
@@ -275,6 +283,11 @@ def build_deployment_readiness_preview(*, service, market_intel_tables, schema_s
phase=service.phase,
)
)
mcp_fetch_candidate_queue_writer_cli_review = (
build_mcp_fetch_candidate_queue_writer_cli_review_preview(
phase=service.phase,
)
)
scheduler_plan = service.build_scheduler_plan()
manual_sample_plan = service.build_manual_sample_plan()
manual_sample_acceptance = service.build_manual_sample_acceptance()
@@ -822,6 +835,33 @@ def build_deployment_readiness_preview(*, service, market_intel_tables, schema_s
and not mcp_fetch_candidate_queue_writer_preflight["file_written"]
and not mcp_fetch_candidate_queue_writer_preflight["scheduler_attached"]
),
"mcp_fetch_candidate_queue_writer_cli_review_preview_safe": bool(
mcp_fetch_candidate_queue_writer_cli_review["mode"]
== "mcp_fetch_candidate_queue_writer_cli_review_preview"
and not mcp_fetch_candidate_queue_writer_cli_review["payload_persisted"]
and not mcp_fetch_candidate_queue_writer_cli_review["cli_review_persisted"]
and not mcp_fetch_candidate_queue_writer_cli_review[
"package_artifact_created"
]
and not mcp_fetch_candidate_queue_writer_cli_review["api_writes_file"]
and not mcp_fetch_candidate_queue_writer_cli_review["api_executes_cli"]
and not mcp_fetch_candidate_queue_writer_cli_review[
"api_reads_approval_token"
]
and not mcp_fetch_candidate_queue_writer_cli_review[
"api_opens_database_connection"
]
and not mcp_fetch_candidate_queue_writer_cli_review["api_writes_database"]
and not mcp_fetch_candidate_queue_writer_cli_review[
"api_uses_external_network"
]
and not mcp_fetch_candidate_queue_writer_cli_review[
"database_write_executed"
]
and not mcp_fetch_candidate_queue_writer_cli_review["cli_executed"]
and not mcp_fetch_candidate_queue_writer_cli_review["file_written"]
and not mcp_fetch_candidate_queue_writer_cli_review["scheduler_attached"]
),
"candidate_queue_writer_postwrite_smoke_planned_safe": bool(
candidate_queue_writer_postwrite_smoke["mode"]
== "candidate_queue_writer_postwrite_smoke_planned"
@@ -1142,6 +1182,7 @@ def build_deployment_readiness_preview(*, service, market_intel_tables, schema_s
"mcp_fetch_candidate_handoff_review": mcp_fetch_candidate_handoff_review,
"mcp_fetch_candidate_queue_review": mcp_fetch_candidate_queue_review,
"mcp_fetch_candidate_queue_writer_preflight": mcp_fetch_candidate_queue_writer_preflight,
"mcp_fetch_candidate_queue_writer_cli_review": mcp_fetch_candidate_queue_writer_cli_review,
"scheduler_plan": scheduler_plan,
"manual_sample_plan": manual_sample_plan,
"manual_sample_acceptance": manual_sample_acceptance,

View File

@@ -0,0 +1,591 @@
"""市場情報 MCP fetch candidate queue writer CLI review preview。
本模組只審核 writer preflight 後的 CLI review 草案API/UI 不執行 CLI、
不讀 approval token、不開 DB、不寫 queue、不掛 scheduler。
"""
from services.market_intel.mcp_fetch_candidate_queue_writer_preflight import (
TARGET_TABLE,
build_mcp_fetch_candidate_queue_writer_preflight_preview,
)
SCRIPT_PATH = "scripts/market_intel_candidate_queue_writer.py"
MAX_CLI_REVIEW_ITEMS = 80
SAFE_REVIEW_MODES = {"candidate_queue_writer_cli_review_preview"}
SAFE_COMMAND_MODES = {"manual_shell_review_only"}
REQUIRED_SAFE_FLAGS = ("--sample-json", "--read-only-preflight")
FORBIDDEN_COMMAND_FLAGS = (
"--apply-real-write",
"--approval-token",
"--execute",
)
FORBIDDEN_SECRET_KEYS = (
"approval_token",
"approval-token",
"api_key",
"authorization",
"bearer",
"client_secret",
"cookie",
"password",
"secret",
"session_cookie",
"token",
)
SAFE_SECRET_METADATA_KEYS = {
"approval_token_shell_only",
"operator_confirmed_no_approval_token_in_payload",
"operator_confirmed_no_secret_payload",
}
FORBIDDEN_RAW_PAYLOAD_KEYS = (
"body_html",
"document_html",
"html",
"page_html",
"raw_body",
"raw_html",
"raw_snapshot",
"response_body",
)
_BLOCKED_SIDE_EFFECT_KEYS = (
"allow_api_execution",
"allow_cli_execution",
"allow_database_write",
"allow_external_network_in_api",
"allow_scheduler_attach",
"api_executed_cli",
"api_executes_cli",
"api_opens_database_connection",
"api_reads_approval_token",
"api_uses_external_network",
"api_writes_database",
"api_writes_file",
"apply_real_write",
"attach_scheduler",
"candidate_queue_created",
"candidate_queue_persisted",
"candidate_review_state_updated",
"cli_executed",
"command_executed",
"database_commit_executed",
"database_connection_opened",
"database_session_created",
"database_write_executed",
"execute_requested",
"external_network_executed",
"file_written",
"network_request_allowed",
"package_artifact_created",
"payload_persisted",
"ready_for_api_database_write",
"ready_for_real_write",
"review_state_update_executed",
"scheduler_attached",
"write_database",
"writes_executed",
"would_write_database",
)
def _as_dict(value):
return value if isinstance(value, dict) else {}
def _as_list(value):
if value is None:
return []
if isinstance(value, (list, tuple, set)):
return list(value)
return [value]
def _safe_int(value):
try:
return int(value or 0)
except (TypeError, ValueError):
return 0
def _safe_text(value, limit=500):
if value is None:
return None
text = str(value).strip()
return text[:limit] if text else None
def _contains_forbidden_key(value, forbidden_keys, *, safe_keys=None):
safe_keys = safe_keys or set()
if isinstance(value, dict):
for key, nested in value.items():
normalized_key = str(key).lower()
if normalized_key in safe_keys and isinstance(nested, bool):
continue
if any(forbidden_key in normalized_key for forbidden_key in forbidden_keys):
return True
if _contains_forbidden_key(nested, forbidden_keys, safe_keys=safe_keys):
return True
elif isinstance(value, list):
return any(
_contains_forbidden_key(item, forbidden_keys, safe_keys=safe_keys)
for item in value
)
return False
def _blocked_side_effects(payload):
found = []
def visit(value, path):
if isinstance(value, dict):
for key, item in value.items():
normalized_key = str(key).lower()
key_path = f"{path}.{key}" if path else key
if normalized_key in _BLOCKED_SIDE_EFFECT_KEYS and bool(item):
found.append(key_path)
visit(item, key_path)
elif isinstance(value, list):
for index, item in enumerate(value):
visit(item, f"{path}[{index}]")
visit(payload, "")
return found
def _preflight_from_inputs(preflight_package, preflight_result, phase):
if isinstance(preflight_result, dict) and preflight_result:
return preflight_result
preflight_package = _as_dict(preflight_package)
return build_mcp_fetch_candidate_queue_writer_preflight_preview(
queue_review_package=(
preflight_package.get("queue_review_package")
or preflight_package.get("candidate_queue_review")
or preflight_package.get("queue_review")
),
queue_review_result=(
preflight_package.get("queue_review_result")
or preflight_package.get("mcp_fetch_candidate_queue_review")
),
writer_preflight=(
preflight_package.get("writer_preflight")
or preflight_package.get("candidate_queue_writer_preflight")
or preflight_package.get("preflight")
),
phase=phase,
)
def _preflight_summary(preflight):
preflight = _as_dict(preflight)
writer_summary = _as_dict(preflight.get("writer_preflight_summary"))
rows = _as_list(writer_summary.get("payload_rows"))
side_effects_clear = bool(
not preflight.get("network_request_allowed")
and not preflight.get("api_executes_cli")
and not preflight.get("api_opens_database_connection")
and not preflight.get("api_writes_database")
and not preflight.get("api_uses_external_network")
and not preflight.get("database_write_executed")
and not preflight.get("cli_executed")
and not preflight.get("file_written")
and not preflight.get("scheduler_attached")
and not preflight.get("candidate_queue_created")
and not preflight.get("candidate_queue_persisted")
and not preflight.get("candidate_review_state_updated")
)
return {
"mode": preflight.get("mode"),
"accepted": bool(
preflight.get("mcp_fetch_candidate_queue_writer_preflight_accepted")
),
"ready_for_candidate_queue_writer_cli_review": bool(
preflight.get("ready_for_candidate_queue_writer_cli_review")
),
"target_table": writer_summary.get("target_table"),
"preflight_id": writer_summary.get("preflight_id"),
"payload_row_count": _safe_int(writer_summary.get("payload_row_count")),
"campaign_candidate_count": _safe_int(
writer_summary.get("campaign_candidate_count")
),
"product_candidate_count": _safe_int(
writer_summary.get("product_candidate_count")
),
"candidate_keys": sorted(
row.get("candidate_key") for row in rows if row.get("candidate_key")
),
"dedupe_keys": sorted(row.get("dedupe_key") for row in rows if row.get("dedupe_key")),
"side_effects_clear": side_effects_clear,
"blocked_reasons": preflight.get("blocked_reasons", []),
}
def _sample_cli_review_package():
preflight_preview = build_mcp_fetch_candidate_queue_writer_preflight_preview()
preflight_package = preflight_preview["sample_writer_preflight_package"]
preflight_result = build_mcp_fetch_candidate_queue_writer_preflight_preview(
queue_review_package=preflight_package["queue_review_package"],
queue_review_result=preflight_package["queue_review_result"],
writer_preflight=preflight_package["writer_preflight"],
)
preflight = _preflight_summary(preflight_result)
return {
"writer_preflight_package": preflight_package,
"writer_preflight_result": preflight_result,
"writer_cli_review": {
"review_id": "market-intel-candidate-writer-cli-review-sample",
"review_mode": "candidate_queue_writer_cli_review_preview",
"command_mode": "manual_shell_review_only",
"script_path": SCRIPT_PATH,
"target_table": TARGET_TABLE,
"preflight_id": preflight["preflight_id"],
"expected_payload_row_count": preflight["payload_row_count"],
"expected_candidate_keys": preflight["candidate_keys"],
"expected_dedupe_keys": preflight["dedupe_keys"],
"command_argv_preview": [
"python3",
SCRIPT_PATH,
"--sample-json",
"<operator-reviewed-json>",
"--read-only-preflight",
],
"operator_confirmed_no_api_cli_execution": True,
"operator_confirmed_no_database_write": True,
"operator_confirmed_no_external_network": True,
"operator_confirmed_no_scheduler_attach": True,
"operator_confirmed_no_persistence": True,
"operator_confirmed_no_approval_token_in_payload": True,
"operator_confirmed_no_secret_payload": True,
"dry_run_first_required": True,
"approval_token_shell_only": True,
"api_execution_allowed": False,
"real_write_allowed": False,
},
}
def _cli_review_summary(cli_review):
cli_review = _as_dict(cli_review)
command_argv = [
_safe_text(item, 200) for item in _as_list(cli_review.get("command_argv_preview"))
]
command_argv = [item for item in command_argv if item]
candidate_keys = sorted(
key for key in _as_list(cli_review.get("expected_candidate_keys")) if key
)
dedupe_keys = sorted(
key for key in _as_list(cli_review.get("expected_dedupe_keys")) if key
)
operator_boundaries_confirmed = bool(
cli_review.get("operator_confirmed_no_api_cli_execution")
and cli_review.get("operator_confirmed_no_database_write")
and cli_review.get("operator_confirmed_no_external_network")
and cli_review.get("operator_confirmed_no_scheduler_attach")
and cli_review.get("operator_confirmed_no_persistence")
and cli_review.get("operator_confirmed_no_approval_token_in_payload")
and cli_review.get("operator_confirmed_no_secret_payload")
)
return {
"provided_keys": sorted(cli_review.keys()),
"review_id": _safe_text(cli_review.get("review_id"), 160),
"review_mode": _safe_text(cli_review.get("review_mode"), 120),
"command_mode": _safe_text(cli_review.get("command_mode"), 120),
"script_path": _safe_text(cli_review.get("script_path"), 240),
"target_table": _safe_text(cli_review.get("target_table"), 160),
"preflight_id": _safe_text(cli_review.get("preflight_id"), 160),
"expected_payload_row_count": _safe_int(
cli_review.get("expected_payload_row_count")
),
"expected_candidate_keys": candidate_keys,
"expected_dedupe_keys": dedupe_keys,
"command_argv_preview": command_argv,
"required_safe_flags_present": all(
flag in command_argv for flag in REQUIRED_SAFE_FLAGS
),
"forbidden_command_flags_absent": not any(
flag in command_argv for flag in FORBIDDEN_COMMAND_FLAGS
),
"command_invokes_expected_script": bool(
SCRIPT_PATH in command_argv and command_argv[:1] in (["python"], ["python3"])
),
"operator_boundaries_confirmed": operator_boundaries_confirmed,
"dry_run_first_required": bool(cli_review.get("dry_run_first_required")),
"approval_token_shell_only": bool(cli_review.get("approval_token_shell_only")),
"api_execution_allowed": bool(cli_review.get("api_execution_allowed")),
"real_write_allowed": bool(cli_review.get("real_write_allowed")),
"candidate_keys_unique": bool(
candidate_keys and len(set(candidate_keys)) == len(candidate_keys)
),
"dedupe_keys_unique": bool(
dedupe_keys and len(set(dedupe_keys)) == len(dedupe_keys)
),
"raw_payload_submitted_to_api": _contains_forbidden_key(
cli_review,
FORBIDDEN_RAW_PAYLOAD_KEYS,
),
"secret_or_token_submitted_to_api": _contains_forbidden_key(
cli_review,
FORBIDDEN_SECRET_KEYS,
safe_keys=SAFE_SECRET_METADATA_KEYS,
),
"blocked_side_effects": _blocked_side_effects(cli_review),
}
def _review_gates(
*,
preflight_received,
cli_review_received,
cli_review_valid_object,
preflight,
cli_review,
):
preflight_candidate_keys = set(preflight["candidate_keys"])
review_candidate_keys = set(cli_review["expected_candidate_keys"])
preflight_dedupe_keys = set(preflight["dedupe_keys"])
review_dedupe_keys = set(cli_review["expected_dedupe_keys"])
return [
{
"key": "writer_preflight_payload_or_result_received",
"label": "已提供 writer preflight package 或已審核結果",
"passed": preflight_received,
},
{
"key": "writer_preflight_accepted",
"label": "writer preflight gate 必須已通過",
"passed": preflight["accepted"],
},
{
"key": "writer_preflight_ready_for_cli_review",
"label": "writer preflight 只放行到 CLI review",
"passed": preflight["ready_for_candidate_queue_writer_cli_review"],
},
{
"key": "writer_preflight_side_effect_free",
"label": "writer preflight 未顯示 API 執行、寫 DB 或掛 scheduler",
"passed": preflight["side_effects_clear"],
},
{
"key": "writer_cli_review_payload_received",
"label": "已提供 writer CLI review 草案",
"passed": cli_review_received,
},
{
"key": "writer_cli_review_valid_object",
"label": "writer CLI review payload 必須是 JSON object",
"passed": cli_review_valid_object,
},
{
"key": "writer_cli_review_identity_recorded",
"label": "writer CLI review 必須記錄 review_id",
"passed": bool(cli_review["review_id"]),
},
{
"key": "writer_cli_review_mode_preview_only",
"label": "writer CLI review 必須維持 preview mode",
"passed": cli_review["review_mode"] in SAFE_REVIEW_MODES,
},
{
"key": "writer_cli_command_mode_manual_shell_only",
"label": "command mode 必須是 manual_shell_review_only",
"passed": cli_review["command_mode"] in SAFE_COMMAND_MODES,
},
{
"key": "writer_cli_script_path_safe",
"label": "script path 必須是候選 queue writer CLI",
"passed": cli_review["script_path"] == SCRIPT_PATH,
},
{
"key": "writer_cli_target_table_safe",
"label": "target table 必須是 market_alert_review_queue",
"passed": cli_review["target_table"] == TARGET_TABLE,
},
{
"key": "writer_cli_preflight_id_matches",
"label": "CLI review preflight_id 必須對齊 writer preflight",
"passed": bool(
cli_review["preflight_id"]
and cli_review["preflight_id"] == preflight["preflight_id"]
),
},
{
"key": "writer_cli_payload_count_matches_preflight",
"label": "CLI review row count 必須對齊 writer preflight",
"passed": bool(
0 < cli_review["expected_payload_row_count"] <= MAX_CLI_REVIEW_ITEMS
and cli_review["expected_payload_row_count"]
== preflight["payload_row_count"]
),
},
{
"key": "writer_cli_candidate_keys_match_preflight",
"label": "CLI review candidate keys 必須完全對齊 writer preflight",
"passed": bool(
preflight_candidate_keys
and review_candidate_keys == preflight_candidate_keys
and cli_review["candidate_keys_unique"]
),
},
{
"key": "writer_cli_dedupe_keys_match_preflight",
"label": "CLI review dedupe keys 必須完全對齊 writer preflight",
"passed": bool(
preflight_dedupe_keys
and review_dedupe_keys == preflight_dedupe_keys
and cli_review["dedupe_keys_unique"]
),
},
{
"key": "writer_cli_command_invokes_expected_script",
"label": "command argv 必須只預覽候選 queue writer script",
"passed": cli_review["command_invokes_expected_script"],
},
{
"key": "writer_cli_required_safe_flags_present",
"label": "command argv 必須包含 sample-json 與 read-only preflight",
"passed": cli_review["required_safe_flags_present"],
},
{
"key": "writer_cli_forbidden_write_flags_absent",
"label": "command argv 不得包含 execute/apply/approval-token",
"passed": cli_review["forbidden_command_flags_absent"],
},
{
"key": "writer_cli_operator_boundaries_confirmed",
"label": "操作員確認 API 不執行 CLI、不讀 token、不寫 DB、不保存、不掛排程",
"passed": cli_review["operator_boundaries_confirmed"],
},
{
"key": "writer_cli_dry_run_first_required",
"label": "正式流程必須先 dry-run / read-only preflight",
"passed": cli_review["dry_run_first_required"],
},
{
"key": "writer_cli_approval_token_shell_only",
"label": "approval token 只能在 shell 提供,不得進 API payload",
"passed": cli_review["approval_token_shell_only"],
},
{
"key": "writer_cli_no_api_execution_or_real_write",
"label": "API review payload 不得允許 CLI 執行或 real write",
"passed": bool(
not cli_review["api_execution_allowed"]
and not cli_review["real_write_allowed"]
),
},
{
"key": "writer_cli_no_raw_payload",
"label": "writer CLI review payload 不得回貼 raw HTML/body/snapshot",
"passed": not cli_review["raw_payload_submitted_to_api"],
},
{
"key": "writer_cli_no_secret_or_token_key",
"label": "writer CLI review payload 不得包含 secret、cookie、password 或 token key",
"passed": not cli_review["secret_or_token_submitted_to_api"],
},
{
"key": "writer_cli_side_effect_free",
"label": "writer CLI review payload 不得要求 API 執行、寫檔、寫 DB 或掛 scheduler",
"passed": not cli_review["blocked_side_effects"],
},
]
def build_mcp_fetch_candidate_queue_writer_cli_review_preview(
*,
writer_preflight_package=None,
writer_preflight_result=None,
writer_cli_review=None,
phase=None,
):
"""建立 fetch candidate queue writer CLI review不執行 CLI 或寫入。"""
writer_preflight_package = _as_dict(writer_preflight_package)
preflight_result_received = bool(
isinstance(writer_preflight_result, dict) and writer_preflight_result
)
cli_review_valid_object = (
isinstance(writer_cli_review, dict) if writer_cli_review is not None else True
)
cli_review_payload = _as_dict(writer_cli_review)
preflight_data = _preflight_from_inputs(
writer_preflight_package,
writer_preflight_result,
phase,
)
preflight_received = bool(
writer_preflight_package or preflight_result_received
)
payload_received = bool(
preflight_received or cli_review_payload or writer_cli_review is not None
)
cli_review_received = bool(cli_review_payload)
preflight = _preflight_summary(preflight_data)
cli_review = _cli_review_summary(cli_review_payload)
gates = _review_gates(
preflight_received=preflight_received,
cli_review_received=cli_review_received,
cli_review_valid_object=cli_review_valid_object,
preflight=preflight,
cli_review=cli_review,
)
blocked_reasons = [gate["key"] for gate in gates if not gate["passed"]]
accepted = bool(payload_received and not blocked_reasons)
return {
"mode": (
"mcp_fetch_candidate_queue_writer_cli_review"
if payload_received
else "mcp_fetch_candidate_queue_writer_cli_review_preview"
),
"phase": phase,
"writer_cli_review_payload_received": payload_received,
"writer_preflight_received": preflight_received,
"writer_cli_review_received": cli_review_received,
"writer_cli_review_valid_object": cli_review_valid_object,
"writer_preflight_accepted": preflight["accepted"],
"mcp_fetch_candidate_queue_writer_cli_review_accepted": accepted,
"candidate_queue_writer_cli_review_ready": accepted,
"ready_for_candidate_queue_writer_run_package_review": accepted,
"ready_for_api_database_write": False,
"ready_for_real_write": False,
"ready_for_scheduler_attach": False,
"network_request_allowed": False,
"api_executes_cli": False,
"api_reads_approval_token": False,
"api_opens_database_connection": False,
"api_writes_database": False,
"api_uses_external_network": False,
"cli_executed": False,
"candidate_queue_created": False,
"candidate_queue_persisted": False,
"candidate_review_state_updated": False,
"payload_row_count": cli_review["expected_payload_row_count"],
"gate_count": len(gates),
"passed_gate_count": sum(1 for gate in gates if gate["passed"]),
"blocked_reasons": blocked_reasons,
"gates": gates,
"writer_preflight_summary": preflight,
"writer_cli_review_summary": cli_review,
"sample_writer_cli_review_package": _sample_cli_review_package(),
"next_operator_steps": [
"CLI review 通過後,只代表可整理 operator run package不代表可執行 CLI",
"真正 --execute、--apply-real-write 與 approval token 必須留在人工 shell gate",
"API/UI 仍不得讀 token、不得執行 CLI、不得開 DB、不得寫 queue",
],
"payload_persisted": False,
"cli_review_persisted": False,
"package_artifact_created": False,
"api_writes_file": False,
"database_session_created": False,
"database_commit_executed": False,
"database_write_executed": False,
"external_network_executed": False,
"file_written": False,
"writes_executed": False,
"would_write_database": False,
"scheduler_attached": False,
}

View File

@@ -1,3 +1,3 @@
"""市場情報 rollout phase 單一來源。"""
MARKET_INTEL_PHASE = "phase_128_market_intel_mcp_fetch_candidate_queue_writer_preflight"
MARKET_INTEL_PHASE = "phase_129_market_intel_mcp_fetch_candidate_queue_writer_cli_review"