feat(smoke): monitor pchome controlled apply final preflight
Some checks failed
CD Pipeline / deploy (push) Has been cancelled

This commit is contained in:
ogt
2026-07-03 15:21:29 +08:00
parent ce8f1d41a3
commit a2d0417a59
6 changed files with 1299 additions and 17 deletions

View File

@@ -107,10 +107,10 @@ def test_system_metrics_exports_scheduled_health_summary():
Gauge,
{
"summary": {
"ok": 21,
"ok": 22,
"warning": 1,
"critical": 0,
"total": 22,
"total": 23,
"primary_human_gate_count": 0,
"writes_database_count": 0,
},
@@ -134,6 +134,7 @@ def test_system_metrics_exports_scheduled_health_summary():
{"key": "pchome_auto_policy_verifier_receipt_closeout", "status": "ok"},
{"key": "pchome_auto_policy_authorization_evidence_execution_preflight", "status": "ok"},
{"key": "pchome_auto_policy_authorization_evidence_execution_closeout", "status": "ok"},
{"key": "pchome_auto_policy_controlled_apply_final_preflight", "status": "ok"},
{"key": "sitewide_ui_ux_agent", "status": "ok"},
{"key": "sitewide_visual_qa", "status": "ok"},
],
@@ -141,7 +142,7 @@ def test_system_metrics_exports_scheduled_health_summary():
)
output = generate_latest(registry).decode("utf-8")
assert 'momo_ai_automation_scheduled_health_summary_total{status="ok"} 21.0' in output
assert 'momo_ai_automation_scheduled_health_summary_total{status="ok"} 22.0' in output
assert 'momo_ai_automation_scheduled_health_summary_total{status="warning"} 1.0' in output
assert (
'momo_ai_automation_scheduled_health_family_status{family="ai_automation_smoke",status="ok"} 1.0'
@@ -219,6 +220,10 @@ def test_system_metrics_exports_scheduled_health_summary():
'momo_ai_automation_scheduled_health_family_status{family="pchome_auto_policy_authorization_evidence_execution_closeout",status="ok"} 1.0'
in output
)
assert (
'momo_ai_automation_scheduled_health_family_status{family="pchome_auto_policy_controlled_apply_final_preflight",status="ok"} 1.0'
in output
)
assert (
'momo_ai_automation_scheduled_health_family_status{family="sitewide_ui_ux_agent",status="ok"} 1.0'
in output

View File

@@ -938,6 +938,95 @@ def _auto_policy_authorization_evidence_execution_closeout_history_check(status=
}
def _auto_policy_controlled_apply_final_preflight_history_check(status="ok"):
return {
"name": "PChome auto-policy controlled apply final preflight",
"status": status,
"summary": (
"PChome auto-policy controlled apply final preflight 已自動完成"
),
"details": {
"policy": (
"read_only_pchome_growth_auto_policy_db_apply_controlled_apply_"
"final_preflight"
),
"result": "DB_APPLY_CONTROLLED_APPLY_FINAL_PREFLIGHT_READY",
"controlled_apply_final_preflight_ready_count": 1,
"controlled_apply_final_preflight_check_count": 12,
"controlled_apply_final_preflight_pass_count": 12,
"controlled_apply_final_preflight_waiting_count": 0,
"authorization_evidence_execution_closeout_ready_count": 1,
"authorization_evidence_execution_closeout_check_count": 12,
"authorization_evidence_execution_preflight_ready_count": 1,
"authorization_evidence_execution_preflight_check_count": 12,
"authorization_verifier_receipt_closeout_ready_count": 1,
"verifier_receipt_closeout_check_count": 12,
"database_apply_final_verifier_gate_count": 1,
"database_apply_authorization_final_verifier_gate_ready_count": 1,
"controlled_apply_final_preflight_count": 1,
"controlled_apply_final_preflight_field_count": 12,
"controlled_apply_final_preflight_acceptance_gate_count": 10,
"rollback_binding_count": 1,
"rollback_binding_field_count": 8,
"post_apply_verifier_binding_count": 1,
"post_apply_verifier_binding_field_count": 8,
"authorization_evidence_execution_closeout_field_count": 12,
"authorization_evidence_execution_closeout_acceptance_gate_count": 10,
"post_apply_verifier_required_count": 1,
"same_run_truth_required_count": 1,
"writes_script_count": 0,
"writes_artifact_count": 0,
"reads_secret_count": 0,
"executes_script_count": 0,
"executes_migration_count": 0,
"executes_endpoint_count": 0,
"executes_sql_count": 0,
"writes_database_count": 0,
"signs_database_apply_authorization_count": 0,
"primary_human_gate_count": 0,
"manual_review_required_count": 0,
"manual_review_mode": "exception_only",
"payload_source": "local_contract_fixture",
"execute_fetch": True,
"outbound_network": False,
"business_data_source": False,
"ready_for_future_database_apply_controlled_apply_final_preflight": True,
"can_enter_future_database_apply_controlled_dry_run_package": True,
"controlled_apply_final_preflight_ready": True,
"rollback_binding_ready": True,
"post_apply_verifier_binding_ready": True,
"rollback_bound": True,
"post_apply_verifier_bound": True,
"dry_run_only": True,
"check_mode_only": True,
"rollback_execution_authorized": False,
"rollback_executes_sql": False,
"rollback_writes_database": False,
"rollback_reads_secret": False,
"rollback_requires_same_run_truth": True,
"post_apply_verifier_must_run_after_apply": True,
"post_apply_verifier_execution_authorized_in_preview": False,
"permits_future_database_apply_controlled_dry_run_package": True,
"accepts_plaintext_secret": False,
"reads_secret_in_preview": False,
"signature_material_included": False,
"secret_material_included": False,
"secret_material_required_in_preview": False,
"executes_authorization_evidence": False,
"executes_database_apply": False,
"executes_endpoint": False,
"executes_sql": False,
"ready_for_database_apply_now": False,
"database_apply_authorized": False,
"issues_database_apply_authorization": False,
"signs_database_apply_authorization": False,
"next_machine_action": (
"continue_to_pchome_auto_policy_controlled_dry_run_package_lane"
),
},
}
def test_event_router_smoke_reports_queued_deliveries(tmp_path, monkeypatch):
from services import ai_automation_metrics as metrics
from services import ai_automation_smoke_service as smoke
@@ -990,6 +1079,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch):
monkeypatch.setattr(smoke, "_pchome_auto_policy_verifier_receipt_closeout_check", lambda: smoke._check("auto-policy verifier receipt closeout", "ok", "ok"))
monkeypatch.setattr(smoke, "_pchome_auto_policy_authorization_evidence_execution_preflight_check", lambda: smoke._check("auto-policy authorization evidence execution preflight", "ok", "ok"))
monkeypatch.setattr(smoke, "_pchome_auto_policy_authorization_evidence_execution_closeout_check", lambda: smoke._check("auto-policy authorization evidence execution closeout", "ok", "ok"))
monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_apply_final_preflight_check", lambda: smoke._check("auto-policy controlled apply final preflight", "ok", "ok"))
monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok"))
monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok"))
monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok"))
@@ -997,7 +1087,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch):
result = smoke.collect_ai_automation_smoke(record_history=False)
assert result["status"] == "critical"
assert result["summary"] == {"ok": 25, "warning": 1, "critical": 1, "total": 27}
assert result["summary"] == {"ok": 26, "warning": 1, "critical": 1, "total": 28}
def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch):
@@ -3163,6 +3253,106 @@ def test_pchome_auto_policy_authorization_evidence_execution_closeout_reports_fi
)
def test_pchome_auto_policy_controlled_apply_final_preflight_reports_bindings_no_apply():
from services import ai_automation_smoke_service as smoke
result = smoke._pchome_auto_policy_controlled_apply_final_preflight_check()
assert result["status"] == "ok"
assert result["details"]["result"] == (
"DB_APPLY_CONTROLLED_APPLY_FINAL_PREFLIGHT_READY"
)
assert result["details"]["controlled_apply_final_preflight_pass_count"] == 12
assert result["details"]["controlled_apply_final_preflight_check_count"] == 12
assert result["details"]["controlled_apply_final_preflight_waiting_count"] == 0
assert (
result["details"]["authorization_evidence_execution_closeout_ready_count"]
== 1
)
assert (
result["details"]["authorization_evidence_execution_closeout_check_count"]
== 12
)
assert result["details"]["database_apply_final_verifier_gate_count"] == 1
assert (
result["details"][
"database_apply_authorization_final_verifier_gate_ready_count"
]
== 1
)
assert result["details"]["controlled_apply_final_preflight_count"] == 1
assert result["details"]["controlled_apply_final_preflight_field_count"] == 12
assert (
result["details"]["controlled_apply_final_preflight_acceptance_gate_count"]
== 10
)
assert result["details"]["rollback_binding_count"] == 1
assert result["details"]["rollback_binding_field_count"] == 8
assert result["details"]["post_apply_verifier_binding_count"] == 1
assert result["details"]["post_apply_verifier_binding_field_count"] == 8
assert result["details"]["post_apply_verifier_required_count"] == 1
assert result["details"]["same_run_truth_required_count"] == 1
assert result["details"]["payload_source"] == "local_contract_fixture"
assert result["details"]["machine_fetch_evidence_source"] == "local_schema_fixture"
assert result["details"]["outbound_network"] is False
assert result["details"]["business_data_source"] is False
assert result["details"]["primary_human_gate_count"] == 0
assert result["details"]["manual_review_required_count"] == 0
assert result["details"]["reads_secret_count"] == 0
assert result["details"]["executes_script_count"] == 0
assert result["details"]["executes_endpoint_count"] == 0
assert result["details"]["executes_sql_count"] == 0
assert result["details"]["writes_database_count"] == 0
assert result["details"]["signs_database_apply_authorization_count"] == 0
assert (
result["details"][
"ready_for_future_database_apply_controlled_apply_final_preflight"
]
is True
)
assert (
result["details"]["can_enter_future_database_apply_controlled_dry_run_package"]
is True
)
assert result["details"]["controlled_apply_final_preflight_ready"] is True
assert result["details"]["rollback_binding_ready"] is True
assert result["details"]["post_apply_verifier_binding_ready"] is True
assert result["details"]["rollback_bound"] is True
assert result["details"]["post_apply_verifier_bound"] is True
assert result["details"]["dry_run_only"] is True
assert result["details"]["check_mode_only"] is True
assert result["details"]["rollback_execution_authorized"] is False
assert result["details"]["rollback_executes_sql"] is False
assert result["details"]["rollback_writes_database"] is False
assert result["details"]["rollback_reads_secret"] is False
assert result["details"]["rollback_requires_same_run_truth"] is True
assert result["details"]["post_apply_verifier_must_run_after_apply"] is True
assert (
result["details"]["post_apply_verifier_execution_authorized_in_preview"]
is False
)
assert (
result["details"]["permits_future_database_apply_controlled_dry_run_package"]
is True
)
assert result["details"]["accepts_plaintext_secret"] is False
assert result["details"]["reads_secret_in_preview"] is False
assert result["details"]["signature_material_included"] is False
assert result["details"]["secret_material_included"] is False
assert result["details"]["secret_material_required_in_preview"] is False
assert result["details"]["executes_authorization_evidence"] is False
assert result["details"]["executes_database_apply"] is False
assert result["details"]["executes_endpoint"] is False
assert result["details"]["executes_sql"] is False
assert result["details"]["ready_for_database_apply_now"] is False
assert result["details"]["database_apply_authorized"] is False
assert result["details"]["issues_database_apply_authorization"] is False
assert result["details"]["signs_database_apply_authorization"] is False
assert result["details"]["next_machine_action"] == (
"continue_to_pchome_auto_policy_controlled_dry_run_package_lane"
)
def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypatch):
from services import ai_automation_smoke_service as smoke
@@ -3193,6 +3383,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat
monkeypatch.setattr(smoke, "_pchome_auto_policy_verifier_receipt_closeout_check", lambda: smoke._check("auto-policy verifier receipt closeout", "ok", "ok"))
monkeypatch.setattr(smoke, "_pchome_auto_policy_authorization_evidence_execution_preflight_check", lambda: smoke._check("auto-policy authorization evidence execution preflight", "ok", "ok"))
monkeypatch.setattr(smoke, "_pchome_auto_policy_authorization_evidence_execution_closeout_check", lambda: smoke._check("auto-policy authorization evidence execution closeout", "ok", "ok"))
monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_apply_final_preflight_check", lambda: smoke._check("auto-policy controlled apply final preflight", "ok", "ok"))
monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok"))
monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok"))
monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok"))
@@ -3251,7 +3442,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects(
json.dumps({
"generated_at": datetime.now().isoformat(timespec="seconds"),
"status": "ok",
"summary": {"ok": 27, "warning": 0, "critical": 0, "total": 27},
"summary": {"ok": 28, "warning": 0, "critical": 0, "total": 28},
"checks": [
{
"name": "PChome 受控落地 drift monitor",
@@ -3282,6 +3473,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects(
_auto_policy_verifier_receipt_closeout_history_check(),
_auto_policy_authorization_evidence_execution_preflight_history_check(),
_auto_policy_authorization_evidence_execution_closeout_history_check(),
_auto_policy_controlled_apply_final_preflight_history_check(),
{
"name": "AI surface HTML readback",
"status": "ok",
@@ -3343,7 +3535,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects(
)
assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary"
assert summary["status"] == "ok"
assert summary["summary"]["total"] == 24
assert summary["summary"]["total"] == 25
assert summary["summary"]["primary_human_gate_count"] == 0
assert summary["summary"]["writes_database_count"] == 0
assert pchome_family["status"] == "ok"
@@ -4737,6 +4929,125 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects(
authorization_evidence_closeout_family["details"]["database_apply_authorized"]
is False
)
controlled_final_preflight_family = next(
item for item in summary["families"]
if item["key"] == "pchome_auto_policy_controlled_apply_final_preflight"
)
assert controlled_final_preflight_family["status"] == "ok"
assert controlled_final_preflight_family["next_machine_action"] == (
"continue_to_pchome_auto_policy_controlled_dry_run_package_lane"
)
assert controlled_final_preflight_family["details"]["next_machine_action"] == (
"continue_to_pchome_auto_policy_controlled_dry_run_package_lane"
)
assert (
controlled_final_preflight_family["details"][
"controlled_apply_final_preflight_pass_count"
]
== 12
)
assert (
controlled_final_preflight_family["details"][
"authorization_evidence_execution_closeout_ready_count"
]
== 1
)
assert (
controlled_final_preflight_family["details"][
"database_apply_authorization_final_verifier_gate_ready_count"
]
== 1
)
assert controlled_final_preflight_family["details"]["rollback_binding_count"] == 1
assert (
controlled_final_preflight_family["details"]["rollback_binding_field_count"]
== 8
)
assert (
controlled_final_preflight_family["details"]["post_apply_verifier_binding_count"]
== 1
)
assert (
controlled_final_preflight_family["details"][
"post_apply_verifier_binding_field_count"
]
== 8
)
assert (
controlled_final_preflight_family["details"]["payload_source"]
== "local_contract_fixture"
)
assert controlled_final_preflight_family["details"]["outbound_network"] is False
assert controlled_final_preflight_family["details"]["business_data_source"] is False
assert (
controlled_final_preflight_family["details"]["primary_human_gate_count"]
== 0
)
assert controlled_final_preflight_family["details"]["reads_secret_count"] == 0
assert controlled_final_preflight_family["details"]["writes_database_count"] == 0
assert (
controlled_final_preflight_family["details"][
"signs_database_apply_authorization_count"
]
== 0
)
assert (
controlled_final_preflight_family["details"][
"ready_for_future_database_apply_controlled_apply_final_preflight"
]
is True
)
assert (
controlled_final_preflight_family["details"][
"can_enter_future_database_apply_controlled_dry_run_package"
]
is True
)
assert (
controlled_final_preflight_family["details"][
"controlled_apply_final_preflight_ready"
]
is True
)
assert controlled_final_preflight_family["details"]["rollback_binding_ready"] is True
assert (
controlled_final_preflight_family["details"]["post_apply_verifier_binding_ready"]
is True
)
assert controlled_final_preflight_family["details"]["rollback_bound"] is True
assert controlled_final_preflight_family["details"]["post_apply_verifier_bound"] is True
assert controlled_final_preflight_family["details"]["dry_run_only"] is True
assert controlled_final_preflight_family["details"]["check_mode_only"] is True
assert (
controlled_final_preflight_family["details"]["rollback_execution_authorized"]
is False
)
assert controlled_final_preflight_family["details"]["rollback_executes_sql"] is False
assert controlled_final_preflight_family["details"]["rollback_writes_database"] is False
assert (
controlled_final_preflight_family["details"][
"post_apply_verifier_execution_authorized_in_preview"
]
is False
)
assert (
controlled_final_preflight_family["details"][
"permits_future_database_apply_controlled_dry_run_package"
]
is True
)
assert (
controlled_final_preflight_family["details"]["executes_database_apply"]
is False
)
assert (
controlled_final_preflight_family["details"]["ready_for_database_apply_now"]
is False
)
assert (
controlled_final_preflight_family["details"]["database_apply_authorized"]
is False
)
surface_family = next(
item for item in summary["families"]
if item["key"] == "ai_surface_html_readback"
@@ -4771,7 +5082,7 @@ def test_scheduled_automation_health_summary_can_use_current_smoke_without_recor
current_smoke = {
"generated_at": "2026-07-02T12:00:00",
"status": "critical",
"summary": {"ok": 26, "warning": 0, "critical": 1, "total": 27},
"summary": {"ok": 27, "warning": 0, "critical": 1, "total": 28},
"checks": [
{
"name": "PChome 受控落地 drift monitor",
@@ -4796,6 +5107,7 @@ def test_scheduled_automation_health_summary_can_use_current_smoke_without_recor
_auto_policy_verifier_receipt_closeout_history_check(),
_auto_policy_authorization_evidence_execution_preflight_history_check(),
_auto_policy_authorization_evidence_execution_closeout_history_check(),
_auto_policy_controlled_apply_final_preflight_history_check(),
{
"name": "AI surface HTML readback",
"status": "ok",
@@ -4851,7 +5163,7 @@ def test_scheduled_automation_health_summary_falls_back_to_visual_qa_artifact(tm
json.dumps({
"generated_at": datetime.now().isoformat(timespec="seconds"),
"status": "ok",
"summary": {"ok": 25, "warning": 0, "critical": 0, "total": 25},
"summary": {"ok": 26, "warning": 0, "critical": 0, "total": 26},
"checks": [],
}, ensure_ascii=False) + "\n",
encoding="utf-8",
@@ -4942,6 +5254,11 @@ def test_scheduled_automation_health_summary_falls_back_to_visual_qa_artifact(tm
"_pchome_auto_policy_authorization_evidence_execution_closeout_check",
lambda: _auto_policy_authorization_evidence_execution_closeout_history_check(),
)
monkeypatch.setattr(
smoke,
"_pchome_auto_policy_controlled_apply_final_preflight_check",
lambda: _auto_policy_controlled_apply_final_preflight_history_check(),
)
monkeypatch.setattr(
surface_service,
"build_sitewide_visual_qa_readback",
@@ -4990,7 +5307,7 @@ def test_scheduled_automation_health_summary_route_returns_compact_payload(tmp_p
json.dumps({
"generated_at": datetime.now().isoformat(timespec="seconds"),
"status": "ok",
"summary": {"ok": 27, "warning": 0, "critical": 0, "total": 27},
"summary": {"ok": 28, "warning": 0, "critical": 0, "total": 28},
"checks": [],
}, ensure_ascii=False) + "\n",
encoding="utf-8",
@@ -5081,6 +5398,11 @@ def test_scheduled_automation_health_summary_route_returns_compact_payload(tmp_p
"_pchome_auto_policy_authorization_evidence_execution_closeout_check",
lambda: _auto_policy_authorization_evidence_execution_closeout_history_check(),
)
monkeypatch.setattr(
smoke,
"_pchome_auto_policy_controlled_apply_final_preflight_check",
lambda: _auto_policy_controlled_apply_final_preflight_history_check(),
)
app = Flask(__name__)
with app.test_request_context(
@@ -5321,6 +5643,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch):
monkeypatch.setattr(smoke, "_pchome_auto_policy_verifier_receipt_closeout_check", lambda: smoke._check("auto-policy verifier receipt closeout", "ok", "ok"))
monkeypatch.setattr(smoke, "_pchome_auto_policy_authorization_evidence_execution_preflight_check", lambda: smoke._check("auto-policy authorization evidence execution preflight", "ok", "ok"))
monkeypatch.setattr(smoke, "_pchome_auto_policy_authorization_evidence_execution_closeout_check", lambda: smoke._check("auto-policy authorization evidence execution closeout", "ok", "ok"))
monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_apply_final_preflight_check", lambda: smoke._check("auto-policy controlled apply final preflight", "ok", "ok"))
monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check(
"Sitewide visual QA readback",
"ok",
@@ -5342,7 +5665,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch):
item for item in result["checks"]
if item["name"] == "Sitewide visual QA readback"
)
assert result["summary"]["total"] == 27
assert result["summary"]["total"] == 28
assert surface_check["status"] == "ok"
assert surface_check["details"]["checked_surface_count"] == 10
assert sitewide_check["status"] == "ok"