wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into codex/p0-006-ai-log-triage-20260709
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Failing after 2m24s
Details
CD Pipeline / build-and-deploy (push) Has been skipped
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details

Browse Source
This commit is contained in:
ogt
2026-07-10 11:48:38 +08:00
parent 1497878b44 f22fa23ac1
commit a15169e85d
7 changed files with 1064 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

476
apps/api/src/services/awoooi_priority_work_order_readback.py
View File

@@ -84,6 +84,16 @@ _AI_AUTOMATION_NODE_RECEIPT_REQUIRED_FIELDS = [
"km_writeback_ref",
"playbook_trust_writeback_ref",
]
_AI_CONTROLLED_REPAIR_LOOP_REQUIRED_FIELDS = [
"target_selector",
"source_truth_diff",
"candidate_action",
"check_mode_or_dry_run",
"controlled_apply_boundary",
"rollback_or_no_write",
"post_verifier",
"km_playbook_trust_writeback",
]
_COMMANDER_INSERTED_REQUIREMENT_WORK_ITEMS: list[dict[str, Any]] = [
{
"id": "CIR-P0-001",
@@ -4996,6 +5006,149 @@ def _stockplatform_ai_recommendations_wait(
)
def _active_p0_blocker_group_counts(active_blockers: list[str]) -> dict[str, int]:
return {
"host_boot_detection": sum(
1 for blocker in active_blockers if blocker in _REBOOT_HOST_BOOT_DETECTION_BLOCKERS
),
"windows99_vmware": sum(
1 for blocker in active_blockers if blocker.startswith("windows99_")
),
"service_data_backup": sum(
1
for blocker in active_blockers
if blocker
in {
"backup_core_green_not_1",
"host_188_service_green_not_1",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"service_green_not_1",
"wazuh_dashboard_degraded",
}
),
"other": sum(
1
for blocker in active_blockers
if blocker not in _REBOOT_HOST_BOOT_DETECTION_BLOCKERS
and not blocker.startswith("windows99_")
and blocker
not in {
"backup_core_green_not_1",
"host_188_service_green_not_1",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"service_green_not_1",
"wazuh_dashboard_degraded",
}
),
}
def _build_current_p0_execution_focus(state: dict[str, Any]) -> dict[str, Any]:
active_blockers = _strings(state.get("active_p0_live_active_blockers"))
group_counts = _active_p0_blocker_group_counts(active_blockers)
service_blocker_count = _int(
state.get("controlled_service_data_backup_blocker_count")
)
service_backup_closed = (
state.get("controlled_service_data_backup_can_clear_blockers") is True
or service_blocker_count == 0
)
locator_dry_run_ready = bool(
state.get("windows99_vmx_source_locator_controlled_relink_dry_run_ready")
is True
or state.get("windows99_vmx_source_locator_candidate_relink_dry_run_ready")
is True
or str(state.get("windows99_vmx_source_locator_status") or "")
== "collector_readback_scoped_relink_dry_run_ready"
)
autostart_check_ready = bool(
state.get("windows99_vmware_autostart_check_mode_package_ready") is True
and state.get("windows99_vmware_autostart_check_mode_allowed") is True
)
ready_packages: list[dict[str, Any]] = []
if locator_dry_run_ready:
ready_packages.append(
{
"id": "windows99_vmx_source_scoped_relink_dry_run",
"status": str(
state.get("windows99_vmx_source_locator_status") or "ready"
),
"target_aliases": _strings(
state.get("windows99_vmx_source_locator_target_aliases")
),
"next_executable_step": str(
state.get("windows99_vmx_source_locator_next_executable_step")
or "rerun_no_secret_collector_and_scorecard"
),
}
)
if autostart_check_ready:
ready_packages.append(
{
"id": "windows99_vmware_autostart_check_mode",
"status": str(
state.get("windows99_vmware_autostart_check_mode_package_status")
or "ready"
),
"target_aliases": _strings(
state.get("windows99_vmware_autostart_check_mode_target_vm_aliases")
),
"next_executable_step": str(
state.get("windows99_vmware_autostart_check_mode_next_executable_step")
or ""
),
}
)
if service_blocker_count and not service_backup_closed:
stage = "service_data_backup_readback_not_green"
lane = "service_data_backup_readback"
next_executable_step = "collect_service_data_backup_readback_no_restart_no_db_write"
elif ready_packages:
stage = "windows99_no_secret_verifier_rerun_ready"
lane = "windows99_vmware_verify_collection"
next_executable_step = "rerun_no_secret_collector_and_scorecard_no_vm_power_change"
elif group_counts["host_boot_detection"]:
stage = "host_probe_verifier_only_required"
lane = "reboot_event_detector_and_host_probe"
next_executable_step = "rerun_reboot_event_detector_and_host_probe_verify_only_no_reboot"
else:
stage = "p0_readback_review_required"
lane = "p0_mainline_readback"
next_executable_step = str(state.get("active_p0_next_safe_action") or "")
focus = {
"schema_version": "awoooi_current_p0_execution_focus_v1",
"workplan_id": str(state.get("active_p0_workplan_id") or ""),
"state": str(state.get("active_p0_state") or ""),
"stage": stage,
"lane": lane,
"primary_blocker": str(state.get("active_p0_primary_blocker") or ""),
"next_executable_step": next_executable_step,
"readiness_percent": _int(state.get("active_p0_readiness_percent")),
"active_blocker_count": len(active_blockers),
"blocker_group_counts": group_counts,
"ready_package_count": len(ready_packages),
"ready_packages": ready_packages,
"no_secret_verifier_ready": bool(ready_packages),
"scoped_relink_dry_run_ready": locator_dry_run_ready,
"autostart_check_mode_ready": autostart_check_ready,
"service_data_backup_closed": service_backup_closed,
"operation_boundaries": {
"secret_value_read": False,
"host_reboot": False,
"vm_power_change": False,
"remote_write": False,
"database_write": False,
"github_api": False,
},
}
return focus
def _apply_parallel_mainline_execution_overlay(payload: dict[str, Any]) -> None:
state = _dict(payload.setdefault("mainline_execution_state", {}))
items = _list(payload.get("commander_inserted_requirement_work_items"))
@@ -5106,6 +5259,8 @@ def _set_rollups_and_summary(
)
active_blockers = _strings(state.get("active_p0_live_active_blockers"))
active_gap_count = _int(state.get("active_p0_immediate_apply_gap_count"))
current_p0_focus = _build_current_p0_execution_focus(state)
payload["current_p0_execution_focus"] = current_p0_focus
payload["rollups"] = {
"completed_p0_count": len(completed),
@@ -5523,9 +5678,29 @@ def _set_rollups_and_summary(
"stale_current_head_metadata_normalized": runtime_source_truth_available,
"stale_cd_runs_reopen_closed_work": False,
"break_glass_reboot_drill_required": True,
"current_p0_execution_focus_stage": current_p0_focus["stage"],
"current_p0_execution_focus_lane": current_p0_focus["lane"],
"current_p0_execution_focus_ready_package_count": current_p0_focus[
"ready_package_count"
],
"current_p0_execution_focus_no_secret_verifier_ready": current_p0_focus[
"no_secret_verifier_ready"
],
}
payload["summary"] = {
"status": str(payload.get("status") or ""),
"current_p0_execution_focus": current_p0_focus,
"current_p0_execution_focus_stage": current_p0_focus["stage"],
"current_p0_execution_focus_lane": current_p0_focus["lane"],
"current_p0_execution_focus_next_executable_step": current_p0_focus[
"next_executable_step"
],
"current_p0_execution_focus_ready_package_count": current_p0_focus[
"ready_package_count"
],
"current_p0_execution_focus_no_secret_verifier_ready": current_p0_focus[
"no_secret_verifier_ready"
],
"active_p0_workplan_id": str(state.get("active_p0_workplan_id") or ""),
"active_p0_state": str(state.get("active_p0_state") or ""),
"active_p0_readiness_percent": _int(
@@ -6688,6 +6863,307 @@ def _apply_ai_automation_node_receipts(payload: dict[str, Any]) -> None:
summary["ai_automation_node_receipt_metadata_only"] = True
summary["ai_automation_node_receipt_lanes"] = lanes
summary["ai_automation_node_receipt_work_items"] = work_items
_apply_ai_controlled_repair_loop_runbook_matrix(payload)
def _contains_any(value: str, needles: tuple[str, ...]) -> bool:
normalized = value.lower()
return any(needle.lower() in normalized for needle in needles)
def _apply_ai_controlled_repair_loop_runbook_matrix(
payload: dict[str, Any],
) -> None:
items = [
_dict(item)
for item in payload.get("commander_inserted_requirement_work_items") or []
if str(_dict(item).get("priority") or "") == "P0"
]
receipts = [_dict(item) for item in payload.get("ai_automation_node_receipts") or []]
ready_receipt_work_items = {
str(item.get("work_item_id") or "")
for item in receipts
if str(item.get("node_status") or "") == "ready"
}
learning_receipt_ready = any(
str(item.get("node_id") or "") == "learning_writeback"
and str(item.get("node_status") or "") == "ready"
for item in receipts
)
post_verifier_receipt_ready = any(
str(item.get("node_id") or "") == "post_verifier"
and str(item.get("node_status") or "") == "ready"
for item in receipts
)
rows: list[dict[str, Any]] = []
for item in sorted(items, key=lambda raw: _int(raw.get("order"))):
item_id = str(item.get("id") or "")
lane = str(item.get("lane") or "")
status = str(item.get("status") or "")
text = " ".join(
str(item.get(key) or "")
for key in (
"id",
"lane",
"request",
"normalized_work_item",
"current_state",
"acceptance",
"next_action",
"mapped_workplan_id",
"visible_surface",
)
)
break_glass_required = lane == "secret_safety" or _contains_any(
text,
(
"secret plaintext",
"plaintext secret",
"private key",
"cookie",
"authorization header",
"明文",
"密碼",
"敏感值",
),
)
fields = {
"target_selector": bool(
item.get("mapped_workplan_id")
or item.get("visible_surface")
or item_id
),
"source_truth_diff": _contains_any(
text,
(
"gitea",
"source",
"repo",
"inventory",
"matrix",
"snapshot",
"evidence",
"readback",
"production",
"證據",
"讀回",
"正式環境",
),
),
"candidate_action": bool(item.get("next_action"))
or item_id in ready_receipt_work_items,
"check_mode_or_dry_run": _contains_any(
text,
(
"check-mode",
"check mode",
"dry-run",
"dry run",
"verifier",
"hidden prompt",
"metadata verifier",
"non-secret",
"只跑",
"乾跑",
"驗證",
),
),
"controlled_apply_boundary": bool(
break_glass_required
or _contains_any(
text,
(
"controlled",
"ai",
"fail-closed",
"no-write",
"no_secret",
"non-secret",
"不讀 secret",
"受控",
"不恢復 generic",
),
)
),
"rollback_or_no_write": bool(
break_glass_required
or _contains_any(
text,
(
"rollback",
"no-write",
"no write",
"no runtime write",
"不入檔",
"不重印",
"不保存",
"不建立",
"不恢復",
"fail-closed",
),
)
),
"post_verifier": bool(
post_verifier_receipt_ready
or _contains_any(
text,
(
"verifier",
"readback",
"smoke",
"api",
"production",
"deploy marker",
"metadata verifier",
"驗證",
"讀回",
"正式環境",
),
)
),
"km_playbook_trust_writeback": bool(
learning_receipt_ready
or _contains_any(
text,
(
"km",
"rag",
"mcp",
"playbook",
"trust",
"writeback",
"learning",
"學習",
"回寫",
),
)
),
}
ready_field_count = sum(1 for ready in fields.values() if ready)
missing_fields = [
field
for field in _AI_CONTROLLED_REPAIR_LOOP_REQUIRED_FIELDS
if not fields[field]
]
controlled_apply_allowed = not break_glass_required and status not in {
"deferred",
}
if break_glass_required:
coverage_state = "break_glass"
elif not missing_fields:
coverage_state = "ready"
elif ready_field_count > 0:
coverage_state = "partial"
else:
coverage_state = "queued"
rows.append(
{
"work_item_id": item_id,
"priority": "P0",
"order": _int(item.get("order")),
"status": status,
"lane": lane,
"mapped_workplan_id": str(item.get("mapped_workplan_id") or ""),
"coverage_state": coverage_state,
"ready_field_count": ready_field_count,
"required_field_count": len(_AI_CONTROLLED_REPAIR_LOOP_REQUIRED_FIELDS),
"missing_fields": missing_fields,
"target_selector_ready": fields["target_selector"],
"source_truth_diff_ready": fields["source_truth_diff"],
"candidate_action_ready": fields["candidate_action"],
"check_mode_or_dry_run_ready": fields["check_mode_or_dry_run"],
"controlled_apply_boundary_ready": fields[
"controlled_apply_boundary"
],
"controlled_apply_allowed": controlled_apply_allowed,
"break_glass_required": break_glass_required,
"rollback_or_no_write_ready": fields["rollback_or_no_write"],
"post_verifier_ready": fields["post_verifier"],
"km_playbook_trust_writeback_ready": fields[
"km_playbook_trust_writeback"
],
"manual_terminal_allowed": False,
"next_action": str(item.get("next_action") or ""),
}
)
required_total = len(rows) * len(_AI_CONTROLLED_REPAIR_LOOP_REQUIRED_FIELDS)
ready_total = sum(_int(row.get("ready_field_count")) for row in rows)
state_counts = {
state: sum(1 for row in rows if row["coverage_state"] == state)
for state in ("ready", "partial", "queued", "break_glass")
}
next_row = next(
(
row
for row in rows
if row["coverage_state"] not in {"ready", "break_glass"}
and row["status"] not in {"done", "deferred"}
),
next(
(
row
for row in rows
if row["coverage_state"] not in {"ready", "break_glass"}
),
rows[0] if rows else {},
),
)
matrix = {
"schema_version": "ai_controlled_repair_loop_runbook_matrix_v1",
"status": "ai_controlled_repair_loop_runbook_matrix_ready",
"metadata_only": True,
"source": _COMMANDER_INSERTED_REQUIREMENT_SOURCE,
"required_fields": _AI_CONTROLLED_REPAIR_LOOP_REQUIRED_FIELDS,
"manual_as_default_terminal_allowed": False,
"hard_blockers_preserved": [
"secret_plaintext",
"destructive_data_change",
"reboot_or_node_drain",
"force_push_or_repo_ref_delete",
"paid_provider_route_change",
],
"rows": rows,
"summary": {
"p0_runbook_count": len(rows),
"ready_runbook_count": state_counts["ready"],
"partial_runbook_count": state_counts["partial"],
"queued_runbook_count": state_counts["queued"],
"break_glass_runbook_count": state_counts["break_glass"],
"controlled_apply_allowed_count": sum(
1 for row in rows if row["controlled_apply_allowed"] is True
),
"manual_terminal_count": sum(
1 for row in rows if row["manual_terminal_allowed"] is True
),
"required_field_total": required_total,
"ready_field_total": ready_total,
"completion_percent": round(
(ready_total / required_total) * 100
)
if required_total
else 0,
"next_work_item_id": str(next_row.get("work_item_id") or ""),
"next_missing_fields": next_row.get("missing_fields") or [],
},
}
payload["ai_controlled_repair_loop_runbook_matrix"] = matrix
summary = _dict(payload.setdefault("summary", {}))
rollups = _dict(payload.setdefault("rollups", {}))
for key, value in matrix["summary"].items():
summary[f"ai_controlled_repair_loop_{key}"] = value
rollups[f"ai_controlled_repair_loop_{key}"] = value
summary["ai_controlled_repair_loop_schema_version"] = matrix["schema_version"]
summary["ai_controlled_repair_loop_metadata_only"] = True
summary["ai_controlled_repair_loop_manual_as_default_terminal_allowed"] = False
rollups["ai_controlled_repair_loop_schema_version"] = matrix["schema_version"]
rollups["ai_controlled_repair_loop_metadata_only"] = True
rollups["ai_controlled_repair_loop_manual_as_default_terminal_allowed"] = False
def _mark_runtime_generated_at(payload: dict[str, Any]) -> None:

104
apps/api/tests/test_awoooi_priority_work_order_readback_api.py
View File

@@ -589,6 +589,53 @@ def test_awoooi_priority_work_order_readback_surfaces_gitea_bundle_truth():
assert "CIR-P0-AILOOP-002" in payload["summary"][
"ai_automation_node_receipt_work_items"
]
runbook_matrix = payload["ai_controlled_repair_loop_runbook_matrix"]
assert runbook_matrix["schema_version"] == (
"ai_controlled_repair_loop_runbook_matrix_v1"
)
assert runbook_matrix["metadata_only"] is True
assert runbook_matrix["manual_as_default_terminal_allowed"] is False
assert runbook_matrix["required_fields"] == [
"target_selector",
"source_truth_diff",
"candidate_action",
"check_mode_or_dry_run",
"controlled_apply_boundary",
"rollback_or_no_write",
"post_verifier",
"km_playbook_trust_writeback",
]
runbook_rows = runbook_matrix["rows"]
runbook_by_id = {item["work_item_id"]: item for item in runbook_rows}
runbook_summary = runbook_matrix["summary"]
assert runbook_summary["p0_runbook_count"] == sum(
1 for item in inserted_items if item["priority"] == "P0"
)
assert runbook_summary["manual_terminal_count"] == 0
assert runbook_summary["controlled_apply_allowed_count"] == sum(
1 for item in runbook_rows if item["controlled_apply_allowed"] is True
)
assert runbook_summary["break_glass_runbook_count"] == sum(
1 for item in runbook_rows if item["coverage_state"] == "break_glass"
)
assert runbook_summary["next_work_item_id"] == "CIR-P0-007"
assert runbook_by_id["CIR-P0-006"]["break_glass_required"] is True
assert runbook_by_id["CIR-P0-006"]["controlled_apply_allowed"] is False
assert runbook_by_id["CIR-P0-007"]["controlled_apply_allowed"] is True
assert runbook_by_id["CIR-P0-007"]["coverage_state"] == "partial"
assert runbook_by_id["CIR-P0-007"]["manual_terminal_allowed"] is False
assert "km_playbook_trust_writeback" in runbook_by_id["CIR-P0-007"][
"missing_fields"
]
assert payload["summary"]["ai_controlled_repair_loop_p0_runbook_count"] == (
runbook_summary["p0_runbook_count"]
)
assert payload["summary"]["ai_controlled_repair_loop_next_work_item_id"] == (
"CIR-P0-007"
)
assert payload["summary"][
"ai_controlled_repair_loop_manual_as_default_terminal_allowed"
] is False
def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_and_windows99(
@@ -712,6 +759,29 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a
assert state["active_p0_next_safe_action"] == (
"rerun_reboot_event_detector_and_host_probe_verify_only_no_reboot"
)
focus = payload["current_p0_execution_focus"]
assert focus["schema_version"] == "awoooi_current_p0_execution_focus_v1"
assert focus["workplan_id"] == "P0-006"
assert focus["stage"] == "windows99_no_secret_verifier_rerun_ready"
assert focus["lane"] == "windows99_vmware_verify_collection"
assert focus["next_executable_step"] == (
"rerun_no_secret_collector_and_scorecard_no_vm_power_change"
)
assert focus["blocker_group_counts"]["host_boot_detection"] > 0
assert focus["blocker_group_counts"]["windows99_vmware"] == 2
assert focus["ready_package_count"] == 2
assert focus["no_secret_verifier_ready"] is True
assert focus["scoped_relink_dry_run_ready"] is True
assert focus["autostart_check_mode_ready"] is True
assert focus["service_data_backup_closed"] is True
assert focus["operation_boundaries"] == {
"secret_value_read": False,
"host_reboot": False,
"vm_power_change": False,
"remote_write": False,
"database_write": False,
"github_api": False,
}
assert (
state["active_p0_projection_source"]
== "reboot_auto_recovery_slo_scorecard"
@@ -961,6 +1031,21 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a
assert payload["summary"]["active_p0_next_safe_action"] == (
"rerun_reboot_event_detector_and_host_probe_verify_only_no_reboot"
)
assert payload["summary"]["current_p0_execution_focus_stage"] == (
"windows99_no_secret_verifier_rerun_ready"
)
assert payload["summary"]["current_p0_execution_focus_lane"] == (
"windows99_vmware_verify_collection"
)
assert payload["summary"]["current_p0_execution_focus_ready_package_count"] == 2
assert (
payload["summary"]["current_p0_execution_focus_no_secret_verifier_ready"]
is True
)
assert payload["rollups"]["current_p0_execution_focus_stage"] == (
"windows99_no_secret_verifier_rerun_ready"
)
assert payload["rollups"]["current_p0_execution_focus_ready_package_count"] == 2
assert (
payload["summary"]["active_p0_projection_source"]
== "reboot_auto_recovery_slo_scorecard"
@@ -1452,6 +1537,25 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot(
assert endpoint_receipts["learning_writeback"]["km_writeback_ref"] == (
"context_receipt:km=1"
)
endpoint_matrix = data["ai_controlled_repair_loop_runbook_matrix"]
endpoint_rows = endpoint_matrix["rows"]
endpoint_row_by_id = {item["work_item_id"]: item for item in endpoint_rows}
assert endpoint_matrix["status"] == (
"ai_controlled_repair_loop_runbook_matrix_ready"
)
assert endpoint_matrix["summary"]["p0_runbook_count"] == sum(
1 for item in inserted_items if item["priority"] == "P0"
)
assert endpoint_matrix["summary"]["manual_terminal_count"] == 0
assert endpoint_matrix["summary"]["next_work_item_id"] == "CIR-P0-007"
assert endpoint_row_by_id["CIR-P0-006"]["coverage_state"] == "break_glass"
assert endpoint_row_by_id["CIR-P0-006"]["controlled_apply_allowed"] is False
assert endpoint_row_by_id["CIR-P0-007"]["coverage_state"] == "partial"
assert endpoint_row_by_id["CIR-P0-007"]["controlled_apply_allowed"] is True
assert data["summary"]["ai_controlled_repair_loop_next_work_item_id"] == (
"CIR-P0-007"
)
assert data["summary"]["ai_controlled_repair_loop_manual_terminal_count"] == 0
def test_controlled_cd_lane_live_metric_readback_falls_back_to_prometheus(