504c66e1f2
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 2m19s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
849 lines
39 KiB
JSON
849 lines
39 KiB
JSON
{
|
||
"schema_version": "sre_k3s_controlled_automation_work_items_v1",
|
||
"governance_version": "global_product_governance_v2",
|
||
"program_id": "AIA-SRE-P0-20260715",
|
||
"generated_at": "2026-07-17T02:00:46+08:00",
|
||
"status": "phase_3_paid_provider_first_canary_failed_safe_rollback_verified",
|
||
"scope_complete": false,
|
||
"current_p0": {
|
||
"id": "AIA-SRE-013",
|
||
"title": "Claude/Gemini protected-secret 受控上線與五路 SRE 效果驗證",
|
||
"phase": "phase_3",
|
||
"terminal_condition": "同一 source SHA 通過 focused/full tests 與 Gitea CD；durable Gate5 authorization receipt 綁定同一 run；protected-secret metadata、GCP-A/GCP-B/host111/Claude/Gemini 同 fixture、Claude/Gemini finalized receipt、latency/token/cost、paired atomic rollback 與 production status readback 全部可驗證；不代表尚未完成的 HolmesGPT 主 pipeline promotion"
|
||
},
|
||
"architecture": {
|
||
"pipeline": [
|
||
"Alert",
|
||
"Canonical Asset Normalize",
|
||
"Typed Domain Router",
|
||
"HolmesGPT Investigator",
|
||
"Ollama RCA / Gemini Critic",
|
||
"Deterministic Policy",
|
||
"Single Controlled Executor",
|
||
"Independent Verifier",
|
||
"Incident Closure + KM/RAG/MCP/PlayBook"
|
||
],
|
||
"cross_domain_fallback_allowed": false,
|
||
"unknown_asset_terminal": "asset_identity_unresolved",
|
||
"circuit_open_terminal": "stay_in_same_domain_and_create_repair_work_item",
|
||
"same_run_identity_required": [
|
||
"trace_id",
|
||
"run_id",
|
||
"work_item_id"
|
||
],
|
||
"provider_failover_is_not_a_pipeline_stage": true
|
||
},
|
||
"provider_policy": {
|
||
"ordered_route": [
|
||
"ollama_gcp_a",
|
||
"ollama_gcp_b",
|
||
"ollama_local",
|
||
"claude",
|
||
"gemini"
|
||
],
|
||
"route_label": "GCP-A -> GCP-B -> host111 Ollama -> Anthropic Claude -> Gemini API",
|
||
"hops": [
|
||
{
|
||
"position": 1,
|
||
"provider": "ollama_gcp_a",
|
||
"identity": "GCP-A"
|
||
},
|
||
{
|
||
"position": 2,
|
||
"provider": "ollama_gcp_b",
|
||
"identity": "GCP-B"
|
||
},
|
||
{
|
||
"position": 3,
|
||
"provider": "ollama_local",
|
||
"identity": "host111"
|
||
},
|
||
{
|
||
"position": 4,
|
||
"provider": "claude",
|
||
"identity": "Anthropic Claude API"
|
||
},
|
||
{
|
||
"position": 5,
|
||
"provider": "gemini",
|
||
"identity": "Gemini API"
|
||
}
|
||
],
|
||
"ollama_role": "primary_rca_and_action_planning",
|
||
"claude_role": "paid_architecture_rca_and_debug_fallback",
|
||
"gemini_role": "final_fallback_and_optional_critic_only",
|
||
"claude_paid_call_allowed": true,
|
||
"gemini_paid_call_allowed": true,
|
||
"paid_execution_mode": "canary",
|
||
"paid_canary_percent": 5,
|
||
"paid_canary_authorization_ref": "AIA-SRE-013-user-approved-20260716",
|
||
"paid_canary_authorization_status": "user_direction_and_durable_gate5_consumed_first_canary_terminal_failed",
|
||
"durable_gate5_authorization": {
|
||
"required": true,
|
||
"binding": [
|
||
"trace_id",
|
||
"run_id",
|
||
"work_item_id",
|
||
"provider_pair",
|
||
"cost_caps",
|
||
"expiry"
|
||
],
|
||
"accepted_source": "durable_operation_log_or_authorization_receipt",
|
||
"free_form_authorization_ref_is_sufficient": false,
|
||
"status": "consumed_terminal_failed",
|
||
"authorization_ref": "019f6c09-e77c-7800-35af-9179358c8f37",
|
||
"run_id": "019f6c09-e77c-7800-35af-9179358c8f37",
|
||
"trace_id": "00-4a83cb2dea7749b398a5e6d6c90737df6ed6c3b8e9534e8b-89789ec6474d4a52-01",
|
||
"approval_durable": true,
|
||
"single_use_claim_consumed": true,
|
||
"terminal_state": "failed",
|
||
"terminal_error_code": "E-PAID-CANARY-VERIFY"
|
||
},
|
||
"paid_canary_scope": "one committed sanitized fixture per provider; no infrastructure remediation",
|
||
"claude_cost_cap_status": "generation_finalized_and_aggregate_run_cap_verified",
|
||
"gemini_cost_cap_status": "failed_generation_estimate_finalized_and_aggregate_run_cap_verified",
|
||
"claude_credential_status": "runtime_secret_reference_generation_authenticated_no_secret_value_read",
|
||
"gemini_credential_status": "runtime_secret_reference_present_but_provider_authentication_rejected_no_secret_value_read",
|
||
"credential_metadata": {
|
||
"claude": {
|
||
"user_supplied_credential_noted": true,
|
||
"raw_value_recorded_in_repository": false,
|
||
"raw_value_readback_allowed": false,
|
||
"protected_secret_reference_status": "k8s_secret_ref_present_and_generation_authenticated"
|
||
},
|
||
"gemini": {
|
||
"user_supplied_credential_noted": true,
|
||
"raw_value_recorded_in_repository": false,
|
||
"raw_value_readback_allowed": false,
|
||
"protected_secret_reference_status": "k8s_secret_ref_present_but_generation_authentication_rejected"
|
||
}
|
||
},
|
||
"cloud_transport_contract": {
|
||
"ollama_gcp_a": {
|
||
"observed_transport": "public_http",
|
||
"execution_scope": "sanitized_candidate_only",
|
||
"tool_loop_terminal": "fail_closed",
|
||
"pre_generation_receipt": "blocked_no_transport_receipt_tcp_timeout",
|
||
"exact_prometheus_target_runtime_status": "missing"
|
||
},
|
||
"ollama_gcp_b": {
|
||
"observed_transport": "public_http",
|
||
"execution_scope": "sanitized_candidate_only",
|
||
"tool_loop_terminal": "fail_closed",
|
||
"pre_generation_receipt": "transport_receipt_present_but_not_same_run_and_public_http_degraded",
|
||
"exact_prometheus_target_runtime_status": "missing"
|
||
},
|
||
"production_sensitive_payload_allowed": false,
|
||
"secure_mesh_or_tls_required_for_promotion": true
|
||
},
|
||
"paid_pair_contract": {
|
||
"enable_and_rollback_as_pair": true,
|
||
"partial_five_lane_terminal": "rollback_both_paid_providers_and_fail_canary",
|
||
"daily_monthly_and_per_incident_cost_caps_required": true,
|
||
"source_status": "same_run_gate5_atomic_single_use_claim_and_aggregate_cap_ready",
|
||
"runtime_status": "first_canary_partial_failed_paired_rollback_verified"
|
||
},
|
||
"latest_canary_receipt": {
|
||
"run_id": "019f6c09-e77c-7800-35af-9179358c8f37",
|
||
"status": "failed_rollback_verified",
|
||
"provider_results": {
|
||
"ollama_gcp_a": "transport_unreachable_no_generation",
|
||
"ollama_gcp_b": "provider_canary_timeout_after_120_seconds",
|
||
"ollama_local": "http_connection_reset_no_generation",
|
||
"claude": "generation_succeeded_contract_verifier_failed",
|
||
"gemini": "authentication_rejected"
|
||
},
|
||
"claude_latency_ms": 4879.9,
|
||
"claude_tokens": 1886,
|
||
"claude_cost_usd": 0.011802,
|
||
"aggregate_accounted_cost_usd": 0.012099,
|
||
"aggregate_cost_cap_usd": 0.25,
|
||
"paired_rollback_verified": true,
|
||
"persistent_paid_routes_enabled": false,
|
||
"raw_prompt_persisted": false,
|
||
"raw_response_persisted": false,
|
||
"secret_value_read_or_returned": false,
|
||
"learning_writeback_status": "pending_provider_scorecard_km_rag_mcp_playbook_ack"
|
||
},
|
||
"production_provider_route_switch_allowed": false,
|
||
"required_before_paid_provider_enablement": [
|
||
"offline_replay_scorecard",
|
||
"shadow_comparison",
|
||
"bounded_canary",
|
||
"daily_and_monthly_cost_cap",
|
||
"per_incident_token_cap",
|
||
"rate_limit_and_circuit_breaker",
|
||
"rollback_to_ollama_chain",
|
||
"production_cost_readback"
|
||
]
|
||
},
|
||
"agent99_host_operations_bridge": {
|
||
"role": "on_prem_policy_controlled_ai_execution_node",
|
||
"single_executor_domains": [
|
||
"windows_vmware",
|
||
"control_plane_recovery"
|
||
],
|
||
"relay_only_domains": [
|
||
"host_systemd",
|
||
"docker_container"
|
||
],
|
||
"dispatch_identity_schema": "agent99_controlled_dispatch_identity_v1",
|
||
"command_envelope_schema": "agent99_controlled_dispatch_input_v1",
|
||
"dispatch_receipt_schema": "agent99_controlled_dispatch_receipt_v1",
|
||
"outcome_schema": "agent99_outcome_contract_v1",
|
||
"completion_callback": "/api/v1/agents/agent99/completion-callback",
|
||
"arbitrary_command_allowed": false,
|
||
"unknown_asset_dispatch_allowed": false,
|
||
"cross_domain_fallback_allowed": false,
|
||
"runtime_terminal": "callback_plus_independent_verifier_plus_learning_writeback"
|
||
},
|
||
"domain_routes": [
|
||
{
|
||
"domain": "kubernetes_workload",
|
||
"executor": "kubernetes_controlled_executor",
|
||
"verifier": "kubernetes_rollout_verifier",
|
||
"fallback": "forbidden"
|
||
},
|
||
{
|
||
"domain": "host_systemd",
|
||
"executor": "host_ansible_executor",
|
||
"verifier": "host_runtime_independent_verifier",
|
||
"fallback": "forbidden"
|
||
},
|
||
{
|
||
"domain": "docker_container",
|
||
"executor": "host_ansible_executor_with_exact_container_playbook",
|
||
"verifier": "container_runtime_independent_verifier",
|
||
"fallback": "forbidden"
|
||
},
|
||
{
|
||
"domain": "windows_vmware",
|
||
"executor": "Agent99",
|
||
"verifier": "agent99_independent_runtime_verifier",
|
||
"fallback": "forbidden"
|
||
},
|
||
{
|
||
"domain": "database",
|
||
"executor": "db_bounded_executor",
|
||
"verifier": "db_independent_verifier",
|
||
"fallback": "forbidden"
|
||
},
|
||
{
|
||
"domain": "backup_restore",
|
||
"executor": "backup_restore_break_glass",
|
||
"verifier": "backup_restore_readback_verifier",
|
||
"fallback": "forbidden",
|
||
"critical_read_only_default": true
|
||
},
|
||
{
|
||
"domain": "unknown",
|
||
"executor": null,
|
||
"verifier": null,
|
||
"terminal": "asset_identity_unresolved",
|
||
"fallback": "forbidden"
|
||
},
|
||
{
|
||
"domain": "control_plane_recovery",
|
||
"executor": "Agent99",
|
||
"verifier": "cold_start_independent_scorecard_verifier",
|
||
"fallback": "forbidden",
|
||
"exact_target_required": true
|
||
}
|
||
],
|
||
"immediate_execution_queue": [
|
||
{
|
||
"position": 1,
|
||
"work_item_id": "AIA-SRE-017",
|
||
"scope": "alert_chain_delivery_truth_and_emergency_card_attribution",
|
||
"blocking_reason": "firing AlertChainBroken invalidates downstream automation and verifier trust",
|
||
"terminal_condition": "Alertmanager monotonic delivery metrics are scraped; Agent99 on host99 independently polls the exact host110 active/critical AlertChain signal without a first-hop credential or raw-payload persistence and relays only the reduced envelope over HTTPS; the false NodePort-counter alert resolves; emergency cards state AI/Agent/executor/verifier truth"
|
||
},
|
||
{
|
||
"position": 2,
|
||
"work_item_id": "AIA-SRE-002",
|
||
"scope": "retire_host110_ollama_identity_and_reconcile_portfolio_inventory",
|
||
"blocking_reason": "stale provider identity produces false failover alerts and corrupts provider comparison",
|
||
"terminal_condition": "Host110 Ollama is a canonical retired tombstone; GCP-A, GCP-B and host111 are direct identities; host111 LaunchAgent is independently verified from host120/121; exact provider sensors are fresh; no host110 Ollama endpoint remains in runtime or monitoring"
|
||
},
|
||
{
|
||
"position": 3,
|
||
"work_item_id": "AIA-SRE-004",
|
||
"scope": "critical_fail_closed_and_exact_alert_chain_domain_route",
|
||
"blocking_reason": "critical or cross-domain candidates must not reach an executor through legacy fallback",
|
||
"terminal_condition": "critical override stays no-write and AlertChainBroken resolves only to host110 Alertmanager container lane"
|
||
},
|
||
{
|
||
"position": 4,
|
||
"work_item_id": "AIA-SRE-013",
|
||
"scope": "five_lane_paid_provider_canary",
|
||
"blocking_reason": "paid calls are allowed only after positions 1-3 have independent readback",
|
||
"terminal_condition": "all five providers return comparable safe receipts; public-cloud prompts remain sanitized and tool loops fail closed; the Gate5 authorization UUID is the exact execution run/trace, has an atomic one-use claim and canary bucket proof; Claude/Gemini use a run-owned lease, paired rollback and verified aggregate cost caps"
|
||
},
|
||
{
|
||
"position": 5,
|
||
"work_item_id": "AIA-SRE-014",
|
||
"scope": "independent_verifier_coverage",
|
||
"blocking_reason": "no apply may close without a domain-specific verifier",
|
||
"terminal_condition": "every mutating catalog and provider lane has an independent terminal readback"
|
||
},
|
||
{
|
||
"position": 6,
|
||
"work_item_id": "AIA-SRE-015",
|
||
"scope": "incident_and_learning_writeback",
|
||
"blocking_reason": "source, CD or Telegram visibility does not prove learning closure",
|
||
"terminal_condition": "same-run incident, KM, RAG, MCP and PlayBook acknowledgements are durable"
|
||
}
|
||
],
|
||
"work_items": [
|
||
{
|
||
"id": "AIA-SRE-001",
|
||
"order": 1,
|
||
"priority": "P0",
|
||
"phase": "phase_1",
|
||
"title": "建立唯一 machine-readable 架構與工作總帳",
|
||
"owner_lane": "AIControlPlane",
|
||
"risk": "low",
|
||
"status": "source_implemented_runtime_pending",
|
||
"dependencies": [],
|
||
"source_refs": [
|
||
"docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json",
|
||
"apps/api/src/services/sre_k3s_controlled_automation_work_items.py"
|
||
],
|
||
"executor": "Gitea controlled CD",
|
||
"verifier": "ledger schema and API readback verifier",
|
||
"rollback": "revert source commit",
|
||
"exit_condition": "production API returns the exact committed ledger and rollups",
|
||
"next_action": "complete loader/API tests then deploy"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-002",
|
||
"order": 2,
|
||
"priority": "P0",
|
||
"phase": "phase_1",
|
||
"title": "Canonical Service Registry 單一來源與 runtime exact mirror",
|
||
"owner_lane": "AssetIdentity",
|
||
"risk": "medium",
|
||
"status": "source_implemented_runtime_pending",
|
||
"dependencies": [
|
||
"AIA-SRE-001"
|
||
],
|
||
"source_refs": [
|
||
"ops/config/service-registry.yaml",
|
||
"ops/monitoring/service-registry.yaml",
|
||
"ops/monitoring/generated/prometheus-scrape-generated.yaml",
|
||
"ops/monitoring/generated/blackbox-targets-generated.yaml",
|
||
"docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json",
|
||
"docs/operations/portfolio-infrastructure-asset-reconciliation-handoff.md",
|
||
"apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py",
|
||
"apps/api/src/api/v1/agents.py",
|
||
"apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py",
|
||
"scripts/ops/retire-host110-ollama-proxy.sh",
|
||
"scripts/ops/render-service-registry-configmap.py",
|
||
"k8s/awoooi-prod/15-service-registry-configmap.yaml"
|
||
],
|
||
"executor": "service registry renderer plus Gitea CD",
|
||
"verifier": "source hash and exact embedded YAML parity",
|
||
"rollback": "reapply previous ConfigMap from prior Gitea SHA",
|
||
"exit_condition": "production ConfigMap, monitoring inventory and portfolio asset graph match live canonical identities; Host110 Ollama is present only as a retired tombstone; GCP-A/GCP-B exact targets and the host111 LaunchAgent sensor are fresh; inventory snapshots are never treated as runtime health",
|
||
"confirmed_truth": [
|
||
"Host110 Ollama has been removed and must remain a retired tombstone, never a fallback endpoint",
|
||
"AIA-SRE-002 run-aia-sre-002-20260716-1752 removed the exact stale Nginx 11435 proxy with rollback backup and independently verified listeners 11435/11434=0, Ollama containers=0, stale config paths=0, nginx active/test pass and Prometheus host110 Ollama targets=0",
|
||
"Host111 is the third provider hop and runs as a macOS LaunchAgent, not systemd",
|
||
"GCP-A and GCP-B source targets are public HTTP sanitized candidates only",
|
||
"gitea-native source target is not production runtime evidence while the legacy GitHub exporter remains visible"
|
||
],
|
||
"runtime_gaps": [
|
||
"Host111 LaunchAgent lacks completed local plus host120/host121 origin readback and a fresh exact sensor series",
|
||
"GCP-A/GCP-B exact Prometheus target series are missing in production",
|
||
"gitea-native exact target is missing and legacy GitHub exporter runtime drift is unresolved"
|
||
],
|
||
"next_action": "deploy the reconciled tombstone/targets, preserve the verified Host110 absence receipt, then verify Host111 LaunchAgent from local and host120/121, GCP-A/GCP-B exact Prometheus series, gitea-native target freshness and legacy exporter absence under one source SHA"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-003",
|
||
"order": 3,
|
||
"priority": "P0",
|
||
"phase": "phase_1",
|
||
"title": "Unknown asset fail-closed 與 deterministic drift work item",
|
||
"owner_lane": "AssetIdentity",
|
||
"risk": "medium",
|
||
"status": "source_implemented_runtime_pending",
|
||
"dependencies": [
|
||
"AIA-SRE-002"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/service_registry.py",
|
||
"apps/api/src/services/auto_repair_service.py"
|
||
],
|
||
"executor": "none for unresolved identity",
|
||
"verifier": "unknown asset replay must return UNRESOLVED and zero candidates",
|
||
"rollback": "not applicable; no-write terminal",
|
||
"exit_condition": "unknown assets create a stable drift ID and never return AUTO/restart fallback",
|
||
"next_action": "production replay one unknown asset without runtime apply"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-004",
|
||
"order": 4,
|
||
"priority": "P0",
|
||
"phase": "phase_1",
|
||
"title": "Typed Domain Router 與跨 domain fallback 禁止",
|
||
"owner_lane": "AutomationRouter",
|
||
"risk": "high",
|
||
"status": "source_implemented_runtime_pending",
|
||
"dependencies": [
|
||
"AIA-SRE-003"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/controlled_alert_target_router.py",
|
||
"apps/api/src/services/awooop_ansible_audit_service.py",
|
||
"apps/api/src/services/awooop_ansible_check_mode_service.py",
|
||
"apps/api/src/api/v1/agents.py",
|
||
"ops/monitoring/alerts-unified.yml"
|
||
],
|
||
"executor": "typed policy router",
|
||
"verifier": "domain/catalog/host/canonical ID scope verifier",
|
||
"rollback": "revert router source while keeping unknown fail-closed",
|
||
"exit_condition": "every candidate is exact-domain scoped; circuit-open never selects another host/domain; GCP-A/GCP-B unsanitized prompts and every cloud tool loop fail closed before a network call",
|
||
"runtime_gaps": [
|
||
"source tests do not prove production router enforcement",
|
||
"Agent99 exact-host Alertmanager pull and reduced HTTPS relay are source-implemented but have no host99 production receipt or primary-path outage verifier"
|
||
],
|
||
"next_action": "run critical override, AlertChainBroken exact-host, cloud sanitized-input/tool-loop fail-closed and full Ansible replay regressions; then verify the same policy in production"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-005",
|
||
"order": 5,
|
||
"priority": "P0",
|
||
"phase": "phase_1",
|
||
"title": "Sentry profiling consumer exact-container bounded recovery",
|
||
"owner_lane": "HostContainerAutomation",
|
||
"risk": "medium",
|
||
"status": "source_implemented_runtime_pending",
|
||
"dependencies": [
|
||
"AIA-SRE-004"
|
||
],
|
||
"source_refs": [
|
||
"infra/ansible/playbooks/110-sentry-profiling-consumer-recovery.yml",
|
||
"apps/api/src/services/awooop_ansible_post_verifier.py"
|
||
],
|
||
"executor": "awoooi-ansible-executor-broker",
|
||
"verifier": "exact container running plus healthy readback",
|
||
"rollback": "stop further writes, same-domain check replay and cooldown",
|
||
"exit_condition": "INC-20260714-471307 replay selects only host_110 exact playbook and post-verifier passes",
|
||
"next_action": "deploy then execute one bounded same-fingerprint controlled replay"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-006",
|
||
"order": 6,
|
||
"priority": "P0",
|
||
"phase": "phase_2",
|
||
"title": "Agent99 Host Operations Bridge typed enforcement",
|
||
"owner_lane": "Agent99",
|
||
"risk": "high",
|
||
"status": "source_implemented_runtime_pending",
|
||
"dependencies": [
|
||
"AIA-SRE-004"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/agent99_sre_bridge.py",
|
||
"apps/api/src/services/agent99_controlled_dispatch_ledger.py",
|
||
"apps/api/src/services/agent99_completion_callback.py",
|
||
"apps/api/src/api/v1/agents.py"
|
||
],
|
||
"executor": "Agent99 for Windows/VMware/control-plane; relay-only elsewhere",
|
||
"verifier": "Agent99 callback plus external same-run verifier",
|
||
"rollback": "disable typed dispatch route and retain no-write Status readback",
|
||
"exit_condition": "all Agent99 dispatches carry canonical typed scope and no legacy catch-all can mutate",
|
||
"next_action": "deploy then replay one allowlisted Host112 or cold-start run plus one cross-domain denied candidate"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-007",
|
||
"order": 7,
|
||
"priority": "P0",
|
||
"phase": "phase_2",
|
||
"title": "K3s workload 專屬 executor 與 rollout verifier",
|
||
"owner_lane": "KubernetesAutomation",
|
||
"risk": "high",
|
||
"status": "planned",
|
||
"dependencies": [
|
||
"AIA-SRE-004"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/executor.py",
|
||
"apps/api/src/services/post_execution_verifier.py"
|
||
],
|
||
"executor": "kubernetes_controlled_executor",
|
||
"verifier": "kubernetes_rollout_verifier",
|
||
"rollback": "declarative rollout undo or previous immutable revision",
|
||
"exit_condition": "K3s incidents never enter Ansible/Agent99 fallback and close with rollout evidence",
|
||
"next_action": "inventory current kubernetes actions and bind allowlisted declarative routes"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-008",
|
||
"order": 8,
|
||
"priority": "P0",
|
||
"phase": "phase_2",
|
||
"title": "Host/systemd 專屬 Ansible executor inventory",
|
||
"owner_lane": "HostAutomation",
|
||
"risk": "high",
|
||
"status": "in_progress",
|
||
"dependencies": [
|
||
"AIA-SRE-004"
|
||
],
|
||
"source_refs": [
|
||
"infra/ansible/inventory/hosts.yml",
|
||
"apps/api/src/services/awooop_ansible_audit_service.py"
|
||
],
|
||
"executor": "host_ansible_executor",
|
||
"verifier": "host_runtime_independent_verifier",
|
||
"rollback": "catalog-specific compensating action",
|
||
"exit_condition": "every host action has one exact inventory host, runtime manager identity, check mode, bounded apply and postcondition; host111 uses its LaunchAgent catalog and is verified locally plus from host120/host121",
|
||
"runtime_gaps": [
|
||
"host111 LaunchAgent catalog and sensor are source candidates without production apply receipt",
|
||
"host120 and host121 independent origin probes have no same-run production verifier receipt"
|
||
],
|
||
"next_action": "close broad catalogs, then check/apply the exact host111 LaunchAgent playbook and verify local, host120 and host121 readbacks plus the canonical sensor rule"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-009",
|
||
"order": 9,
|
||
"priority": "P0",
|
||
"phase": "phase_2",
|
||
"title": "Windows/VMware Agent99 single-executor closure",
|
||
"owner_lane": "Agent99",
|
||
"risk": "high",
|
||
"status": "in_progress",
|
||
"dependencies": [
|
||
"AIA-SRE-006"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/agent99_controlled_dispatch_ledger.py",
|
||
"apps/api/src/services/agent99_same_run_reconcile.py"
|
||
],
|
||
"executor": "Agent99",
|
||
"verifier": "agent99_independent_runtime_verifier",
|
||
"rollback": "allowlisted Status/no-write reconciliation and bounded generation retry",
|
||
"exit_condition": "Windows/VMware runs use durable identity, callback, verifier and learning receipt",
|
||
"next_action": "reconcile remaining Agent99 enterprise work ledger with this program"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-010",
|
||
"order": 10,
|
||
"priority": "P0",
|
||
"phase": "phase_3",
|
||
"title": "DB 專屬 verifier 與 bounded executor",
|
||
"owner_lane": "DatabaseAutomation",
|
||
"risk": "high",
|
||
"status": "source_implemented_runtime_pending",
|
||
"dependencies": [
|
||
"AIA-SRE-004"
|
||
],
|
||
"source_refs": [
|
||
"ops/config/service-registry.yaml",
|
||
"apps/api/src/db/base.py",
|
||
"apps/api/src/core/unit_of_work.py",
|
||
"apps/api/src/services/db_bounded_executor.py",
|
||
"apps/api/src/services/executor_trust_boundary_readback.py",
|
||
"agent99-db-bounded-executor.ps1",
|
||
"scripts/ops/awoooi-workload-db-identity-bootstrap.sh"
|
||
],
|
||
"executor": "db_bounded_executor",
|
||
"verifier": "db_independent_verifier",
|
||
"rollback": "transactional or compensating action only; destructive restore remains break-glass",
|
||
"exit_condition": "DB incidents use DB-native preflight/postconditions and never generic host restart",
|
||
"runtime_entrypoint_contract": {
|
||
"status": "source_ready_runtime_not_applied",
|
||
"command_id": "telegram_receipt_index_apply_v1",
|
||
"canonical_asset": "public.awooop_outbound_message",
|
||
"execution_hub": "host:192.168.0.99",
|
||
"dispatch_role": "Agent99_dispatch_only",
|
||
"executor_domain": "database",
|
||
"runtime_transport": "host99_to_host110_to_host120_fixed_sudo_kubectl_exec",
|
||
"host120_installer_required": false,
|
||
"check_apply_pod_role": "first_ready_awoooi_api_pod",
|
||
"independent_verifier_role": "second_ready_awoooi_api_pod_second_process",
|
||
"arbitrary_sql_allowed": false,
|
||
"cross_domain_fallback_allowed": false
|
||
},
|
||
"next_action": "deploy the fixed Agent99 dispatch through the atomic runtime bundle, then use the already allowlisted host120 sudo kubectl path for one same-trace check/apply/second-pod verify of telegram_receipt_index_apply_v1; separately retain cap-2 global session budget and role headroom >= 8 verification"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-011",
|
||
"order": 11,
|
||
"priority": "P0",
|
||
"phase": "phase_3",
|
||
"title": "Backup/restore readback 與 critical break-glass",
|
||
"owner_lane": "DataProtection",
|
||
"risk": "critical",
|
||
"status": "policy_active",
|
||
"dependencies": [
|
||
"AIA-SRE-004"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/backup_restore_signal_automation.py"
|
||
],
|
||
"executor": "backup_restore_break_glass",
|
||
"verifier": "backup_restore_readback_verifier",
|
||
"rollback": "no-write default; destructive action requires incident-specific break-glass",
|
||
"exit_condition": "freshness, offsite, escrow and restore-drill evidence close without false green",
|
||
"next_action": "bind typed route to DR scorecard without running backup/restore"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-012",
|
||
"order": 12,
|
||
"priority": "P0",
|
||
"phase": "phase_3",
|
||
"title": "HolmesGPT Investigator shadow integration",
|
||
"owner_lane": "Investigator",
|
||
"risk": "high",
|
||
"status": "planned",
|
||
"dependencies": [
|
||
"AIA-SRE-004"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/pre_decision_investigator.py"
|
||
],
|
||
"executor": "none in shadow",
|
||
"verifier": "offline replay quality and prompt-injection safety scorecard",
|
||
"rollback": "remove shadow consumer; active PreDecisionInvestigator remains",
|
||
"exit_condition": "internal immutable artifact passes replay, shadow and canary before core replacement",
|
||
"next_action": "source HolmesGPT from approved internal mirror without GitHub access"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-013",
|
||
"order": 13,
|
||
"priority": "P0",
|
||
"phase": "phase_3",
|
||
"title": "GCP-A/GCP-B/host111 Ollama、Claude、Gemini RCA/critic 與付費 canary/cost gate",
|
||
"owner_lane": "ModelRouter",
|
||
"risk": "critical",
|
||
"status": "source_implemented_runtime_pending",
|
||
"dependencies": [
|
||
"AIA-SRE-012"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/ai_router.py",
|
||
"apps/api/src/services/ollama_failover_manager.py",
|
||
"apps/api/src/services/ai_providers/claude.py",
|
||
"apps/api/src/services/ai_providers/gemini.py",
|
||
"apps/api/src/services/ai_provider_policy.py",
|
||
"apps/api/src/services/failover_alerter.py",
|
||
"apps/api/src/services/cloud_transport_receipt.py",
|
||
"apps/api/src/services/paid_provider_canary_authorization.py",
|
||
"apps/api/src/services/paid_provider_canary_gate5_run.py",
|
||
"apps/api/src/services/paid_provider_canary_validation.py",
|
||
"apps/api/src/jobs/paid_provider_canary_worker.py",
|
||
"apps/api/src/services/ai_rate_limiter.py",
|
||
"scripts/ops/run-paid-provider-canary.py",
|
||
"k8s/awoooi-prod/04-configmap.yaml",
|
||
"k8s/awoooi-prod/06-deployment-api.yaml"
|
||
],
|
||
"executor": "singleton signal-worker paid canary executor plus ordered model router",
|
||
"verifier": "provider-attributed replay accuracy, latency and cost readback",
|
||
"rollback": "Claude/Gemini are disabled together on any partial five-lane result; remain on the ordered Ollama chain",
|
||
"exit_condition": "provider order is enforced; GCP public-HTTP hops remain sanitized candidate-only with cloud tool loops fail-closed until secure transport promotion; paid providers remain 5 percent canary only; the durable Gate5 UUID is the same execution run/trace and is atomically single-use; protected-secret metadata, paired rollback, aggregate cost cap, finalized receipts, latency/token/cost and production readback are verified without secret-value exposure",
|
||
"confirmed_truth": [
|
||
"The user supplied Claude and Gemini credential material, but raw values are not repository data and must not be copied or read back",
|
||
"GCP-A/GCP-B are currently public HTTP boundaries and cannot receive unsanitized or tool-loop payloads",
|
||
"Claude and Gemini enablement, canary outcome and rollback form one atomic pair",
|
||
"Gate5 run 019f6c09-e77c-7800-35af-9179358c8f37 was durably approved and consumed exactly once; Claude generation authenticated, Gemini authentication was rejected, and both paid gates returned to disabled",
|
||
"The first canary accounted USD 0.012099 under the USD 0.25 run cap and persisted neither raw prompt nor raw response",
|
||
"The first Gate5 approval emitted approval_hmac_key_not_set_using_dev_fallback; this is a production approval-signing contract drift, not evidence that paid providers are hard locked"
|
||
],
|
||
"runtime_gaps": [
|
||
"GCP-A transport is unreachable and has no same-run transport receipt",
|
||
"GCP-B is reachable and has qwen3:14b, but the first canary used an obsolete 120 second outer timeout instead of the 300 second diagnose budget",
|
||
"host111 accepts TCP but resets the Ollama HTTP request and requires Agent99 bounded host service diagnosis",
|
||
"Claude authenticated but its first response did not satisfy the deterministic OpenClawDecision contract; strict tool schema source fix awaits CD and replay",
|
||
"Gemini production secret reference is present but provider authentication is rejected; the user-supplied replacement is not yet present in the protected runtime secret path",
|
||
"APPROVAL_HMAC_KEY is not bound to a dedicated protected production secret; the development fallback must be retired after a secret-metadata-only preflight and before production provider promotion",
|
||
"singleton approved-run executor handoff and container import-path fixes await Gitea CD",
|
||
"provider scorecard KM/RAG/MCP/PlayBook acknowledgement is pending"
|
||
],
|
||
"next_action": "deploy the executor, strict Claude schema and Ollama timeout fixes; bind APPROVAL_HMAC_KEY through a dedicated protected secret reference without reading its value; repair GCP-A transport and host111 through their typed owner lanes; rotate Gemini through protected secret ingress without repository or log exposure; then create a new single-use Gate5 run and rerun the sanitized five-lane verifier"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-014",
|
||
"order": 14,
|
||
"priority": "P0",
|
||
"phase": "phase_4",
|
||
"title": "Independent verifier registry 完整覆蓋",
|
||
"owner_lane": "PostExecutionVerifier",
|
||
"risk": "high",
|
||
"status": "in_progress",
|
||
"dependencies": [
|
||
"AIA-SRE-005",
|
||
"AIA-SRE-007",
|
||
"AIA-SRE-008",
|
||
"AIA-SRE-009",
|
||
"AIA-SRE-010"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/awooop_ansible_post_verifier.py",
|
||
"apps/api/src/services/post_execution_verifier.py"
|
||
],
|
||
"executor": "none",
|
||
"verifier": "domain-specific external readback",
|
||
"rollback": "failed verifier triggers same-domain retry or compensating action",
|
||
"exit_condition": "100 percent of mutating catalogs have independent production postconditions; host111 includes local plus host120/host121 origin verification and provider monitoring requires exact fresh series",
|
||
"runtime_gaps": [
|
||
"source-level verifier definitions are not production verifier receipts",
|
||
"Host111 LaunchAgent, GCP provider targets, gitea-native metrics and the host99 Alertmanager pull/relay remain without independent runtime closure"
|
||
],
|
||
"next_action": "reconcile catalog IDs against postcondition registry, then attach same-run runtime receipts for Host111 origins, GCP exact targets, gitea-native metrics and the host99 Alertmanager pull/relay"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-015",
|
||
"order": 15,
|
||
"priority": "P0",
|
||
"phase": "phase_4",
|
||
"title": "Incident closure 與 KM/RAG/MCP/PlayBook durable writeback",
|
||
"owner_lane": "LearningControlPlane",
|
||
"risk": "medium",
|
||
"status": "in_progress",
|
||
"dependencies": [
|
||
"AIA-SRE-014"
|
||
],
|
||
"source_refs": [
|
||
"apps/api/src/services/learning_service.py",
|
||
"apps/api/src/services/awooop_ansible_learning_writeback.py"
|
||
],
|
||
"executor": "learning writeback consumer",
|
||
"verifier": "durable same-run acknowledgement readback",
|
||
"rollback": "append-only correction event; immutable receipts are not overwritten",
|
||
"exit_condition": "closure requires all learning acknowledgements on the same run",
|
||
"next_action": "bind typed domain and verifier outcome into immutable learning receipt"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-016",
|
||
"order": 16,
|
||
"priority": "P1",
|
||
"phase": "phase_5",
|
||
"title": "24h replay、MTTA/MTTR、誤判、復發與成本 scorecard",
|
||
"owner_lane": "AutomationSLO",
|
||
"risk": "low",
|
||
"status": "planned",
|
||
"dependencies": [
|
||
"AIA-SRE-015"
|
||
],
|
||
"source_refs": [],
|
||
"executor": "offline replay worker",
|
||
"verifier": "production aggregate versus same-run receipts",
|
||
"rollback": "read-only scorecard",
|
||
"exit_condition": "all lanes publish MTTA, MTTR, false positive, recurrence, human intervention, verifier and rollback metrics",
|
||
"next_action": "replay the latest 24h alert/runs corpus after phase 1 deploy"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-017",
|
||
"order": 17,
|
||
"priority": "P1",
|
||
"phase": "phase_5",
|
||
"title": "Telegram 與 cockpit 只呈現 canonical lifecycle truth",
|
||
"owner_lane": "ProductExperience",
|
||
"risk": "medium",
|
||
"status": "in_progress",
|
||
"dependencies": [
|
||
"AIA-SRE-015"
|
||
],
|
||
"source_refs": [
|
||
"ops/config/telegram-routing-registry.yaml",
|
||
"ops/alertmanager/alertmanager.yml",
|
||
"k8s/monitoring/prometheus.yml",
|
||
"ops/monitoring/alerts-unified.yml",
|
||
"scripts/ops/deploy-alertmanager-runtime-scrape.sh",
|
||
"agent99-alertmanager-alertchain-poll.ps1",
|
||
"agent99-sre-alert-relay.ps1",
|
||
"scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh",
|
||
"apps/api/src/services/failover_alerter.py",
|
||
"apps/api/src/services/telegram_gateway.py"
|
||
],
|
||
"executor": "canonical Telegram gateway",
|
||
"verifier": "delivery receipt plus desktop/mobile visible smoke",
|
||
"rollback": "suppress duplicate projection; retain incident ledger",
|
||
"exit_condition": "no alert is sprayed across groups/bots; every card states AI/provider/Agent action, executor, verifier and receipt truth; AlertChainBroken uses Alertmanager integration-scoped delivery counters stamped with the independently tested sole-webhook receiver contract plus host99 Agent99 exact-host read-only polling independent of the broken webhook; UI status polling never emits failover alerts",
|
||
"runtime_gaps": [
|
||
"the host99 Agent99 independent pull/reduced relay has no production deployment, freshness, dedupe or controlled-repair receipt",
|
||
"production Alertmanager exposes integration-only notification counters; the bounded check now passes after adding a scrape-time receiver_contract label, but apply, alert resolution and same-run learning receipts remain pending"
|
||
],
|
||
"next_action": "deploy the production-schema-compatible Alertmanager runtime scrape/canonical rules, the host99 Agent99 exact-host poller and the fixed DB dispatch entrypoint through the atomic 17-file bundle; verify receiver-contract freshness, dedupe and the current firing alert resolution; then reconcile every project/product/alert class against its typed domain and canonical Telegram destination"
|
||
},
|
||
{
|
||
"id": "AIA-SRE-018",
|
||
"order": 18,
|
||
"priority": "P0",
|
||
"phase": "phase_6",
|
||
"title": "Gitea CD、production runtime 與 visible closure",
|
||
"owner_lane": "ReleaseControl",
|
||
"risk": "high",
|
||
"status": "planned",
|
||
"dependencies": [
|
||
"AIA-SRE-001",
|
||
"AIA-SRE-002",
|
||
"AIA-SRE-003",
|
||
"AIA-SRE-004",
|
||
"AIA-SRE-005",
|
||
"AIA-SRE-006",
|
||
"AIA-SRE-014",
|
||
"AIA-SRE-015"
|
||
],
|
||
"source_refs": [
|
||
".gitea/workflows/cd.yaml",
|
||
"docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json"
|
||
],
|
||
"executor": "Gitea controlled CD",
|
||
"verifier": "deploy marker, image SHA, API readback and desktop/mobile smoke",
|
||
"rollback": "previous immutable image revision and ConfigMap source SHA",
|
||
"exit_condition": "source/test/CD/runtime/UI evidence all match one deployed SHA; gitea-native exact metrics are fresh and the retired legacy GitHub exporter is absent from production",
|
||
"runtime_gaps": [
|
||
"no program item is runtime closed for this source SHA",
|
||
"gitea-native exact target is missing while legacy GitHub exporter runtime drift remains",
|
||
"source/test readiness, CD terminal, production runtime and visible UI evidence are separate incomplete layers"
|
||
],
|
||
"next_action": "commit phase 1, normal push after shared writer release, then terminal CD plus production API/runtime/UI readback and exact gitea-native/legacy-exporter verification"
|
||
}
|
||
],
|
||
"rollups": {
|
||
"total_items": 18,
|
||
"by_priority": {
|
||
"P0": 16,
|
||
"P1": 2
|
||
},
|
||
"by_status": {
|
||
"source_implemented_runtime_pending": 8,
|
||
"in_progress": 5,
|
||
"planned": 4,
|
||
"policy_active": 1
|
||
},
|
||
"source_implemented_items": 8,
|
||
"runtime_closed_items": 0,
|
||
"program_completion_percent": 0,
|
||
"asset_coverage_status": "partial",
|
||
"runtime_closure_status": "not_started_for_this_program_sha"
|
||
},
|
||
"completion_contract": {
|
||
"required_stages": [
|
||
"sensor_source_receipt",
|
||
"canonical_asset_identity",
|
||
"source_truth_diff",
|
||
"ai_decision_candidate",
|
||
"risk_policy_decision",
|
||
"check_mode_receipt",
|
||
"bounded_execution_receipt",
|
||
"independent_verifier_or_no_write_terminal",
|
||
"incident_closure",
|
||
"telegram_receipt",
|
||
"km_rag_mcp_playbook_writeback_ack"
|
||
],
|
||
"terminal_without_all_stages": "partial_or_degraded_with_safe_next_action",
|
||
"source_test_cd_green_is_runtime_closure": false,
|
||
"evidence_layers": {
|
||
"source_test": "partial_source_evidence_only",
|
||
"gitea_cd_deploy_marker": "pending_for_this_program_sha",
|
||
"production_runtime": "zero_work_items_closed",
|
||
"visible_ui_telegram": "pending_same_run_receipts",
|
||
"learning_writeback": "pending_same_run_durable_acknowledgements"
|
||
},
|
||
"promotion_rule": "a lower evidence layer cannot promote an incomplete higher evidence layer"
|
||
}
|
||
}
|