wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat(automation): enforce typed SRE execution domains

Browse Source
This commit is contained in:
ogt
2026-07-15 12:07:05 +08:00
parent 8258f3f6dd
commit babbb21fb5
21 changed files with 2539 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

562
docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json Normal file
View File

@@ -0,0 +1,562 @@
{
"schema_version": "sre_k3s_controlled_automation_work_items_v1",
"governance_version": "global_product_governance_v2",
"program_id": "AIA-SRE-P0-20260715",
"generated_at": "2026-07-15T15:30:00+08:00",
"status": "phase_1_canonical_identity_and_typed_routing_in_progress",
"scope_complete": false,
"current_p0": {
"id": "AIA-SRE-004",
"title": "Canonical Asset Normalize 與 Typed Domain Router production closure",
"phase": "phase_1",
"terminal_condition": "同一 source SHA 通過 focused/full tests、Gitea CD、production registry parity、typed route replay 與 Sentry exact-container verifier readback"
},
"architecture": {
"pipeline": [
"Alert",
"Canonical Asset Normalize",
"Typed Domain Router",
"HolmesGPT Investigator",
"Ollama RCA / Gemini Critic",
"Deterministic Policy",
"Single Controlled Executor",
"Independent Verifier",
"Incident Closure + KM/RAG/MCP/PlayBook"
],
"cross_domain_fallback_allowed": false,
"unknown_asset_terminal": "asset_identity_unresolved",
"circuit_open_terminal": "stay_in_same_domain_and_create_repair_work_item",
"same_run_identity_required": [
"trace_id",
"run_id",
"work_item_id"
]
},
"provider_policy": {
"ordered_route": [
"ollama_gcp_a",
"ollama_gcp_b",
"ollama_local_111",
"gemini_api"
],
"ollama_role": "primary_rca_and_action_planning",
"gemini_role": "final_fallback_and_optional_critic_only",
"gemini_paid_call_allowed": false,
"gemini_cost_cap_status": "explicit_cost_cap_not_approved",
"gemini_credential_status": "runtime_secret_reference_verifier_pending_no_secret_value_read",
"production_provider_route_switch_allowed": false,
"required_before_gemini_enablement": [
"offline_replay_scorecard",
"shadow_comparison",
"bounded_canary",
"daily_and_monthly_cost_cap",
"per_incident_token_cap",
"rate_limit_and_circuit_breaker",
"rollback_to_ollama_chain",
"production_cost_readback"
]
},
"agent99_host_operations_bridge": {
"role": "on_prem_policy_controlled_ai_execution_node",
"single_executor_domains": [
"windows_vmware",
"control_plane_recovery"
],
"relay_only_domains": [
"host_systemd",
"docker_container"
],
"dispatch_identity_schema": "agent99_controlled_dispatch_identity_v1",
"command_envelope_schema": "agent99_controlled_dispatch_input_v1",
"dispatch_receipt_schema": "agent99_controlled_dispatch_receipt_v1",
"outcome_schema": "agent99_outcome_contract_v1",
"completion_callback": "/api/v1/agents/agent99/completion-callback",
"arbitrary_command_allowed": false,
"unknown_asset_dispatch_allowed": false,
"cross_domain_fallback_allowed": false,
"runtime_terminal": "callback_plus_independent_verifier_plus_learning_writeback"
},
"domain_routes": [
{
"domain": "kubernetes_workload",
"executor": "kubernetes_controlled_executor",
"verifier": "kubernetes_rollout_verifier",
"fallback": "forbidden"
},
{
"domain": "host_systemd",
"executor": "host_ansible_executor",
"verifier": "host_runtime_independent_verifier",
"fallback": "forbidden"
},
{
"domain": "docker_container",
"executor": "host_ansible_executor_with_exact_container_playbook",
"verifier": "container_runtime_independent_verifier",
"fallback": "forbidden"
},
{
"domain": "windows_vmware",
"executor": "Agent99",
"verifier": "agent99_independent_runtime_verifier",
"fallback": "forbidden"
},
{
"domain": "database",
"executor": "db_bounded_executor",
"verifier": "db_independent_verifier",
"fallback": "forbidden"
},
{
"domain": "backup_restore",
"executor": "backup_restore_break_glass",
"verifier": "backup_restore_readback_verifier",
"fallback": "forbidden",
"critical_read_only_default": true
},
{
"domain": "unknown",
"executor": null,
"verifier": null,
"terminal": "asset_identity_unresolved",
"fallback": "forbidden"
}
],
"work_items": [
{
"id": "AIA-SRE-001",
"order": 1,
"priority": "P0",
"phase": "phase_1",
"title": "建立唯一 machine-readable 架構與工作總帳",
"owner_lane": "AIControlPlane",
"risk": "low",
"status": "source_implemented_runtime_pending",
"dependencies": [],
"source_refs": [
"docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json",
"apps/api/src/services/sre_k3s_controlled_automation_work_items.py"
],
"executor": "Gitea controlled CD",
"verifier": "ledger schema and API readback verifier",
"rollback": "revert source commit",
"exit_condition": "production API returns the exact committed ledger and rollups",
"next_action": "complete loader/API tests then deploy"
},
{
"id": "AIA-SRE-002",
"order": 2,
"priority": "P0",
"phase": "phase_1",
"title": "Canonical Service Registry 單一來源與 runtime exact mirror",
"owner_lane": "AssetIdentity",
"risk": "medium",
"status": "source_implemented_runtime_pending",
"dependencies": [
"AIA-SRE-001"
],
"source_refs": [
"ops/config/service-registry.yaml",
"scripts/ops/render-service-registry-configmap.py",
"k8s/awoooi-prod/15-service-registry-configmap.yaml"
],
"executor": "service registry renderer plus Gitea CD",
"verifier": "source hash and exact embedded YAML parity",
"rollback": "reapply previous ConfigMap from prior Gitea SHA",
"exit_condition": "production ConfigMap source hash and API identity lookup match Gitea main",
"next_action": "deploy and read back ConfigMap annotation"
},
{
"id": "AIA-SRE-003",
"order": 3,
"priority": "P0",
"phase": "phase_1",
"title": "Unknown asset fail-closed 與 deterministic drift work item",
"owner_lane": "AssetIdentity",
"risk": "medium",
"status": "source_implemented_runtime_pending",
"dependencies": [
"AIA-SRE-002"
],
"source_refs": [
"apps/api/src/services/service_registry.py",
"apps/api/src/services/auto_repair_service.py"
],
"executor": "none for unresolved identity",
"verifier": "unknown asset replay must return UNRESOLVED and zero candidates",
"rollback": "not applicable; no-write terminal",
"exit_condition": "unknown assets create a stable drift ID and never return AUTO/restart fallback",
"next_action": "production replay one unknown asset without runtime apply"
},
{
"id": "AIA-SRE-004",
"order": 4,
"priority": "P0",
"phase": "phase_1",
"title": "Typed Domain Router 與跨 domain fallback 禁止",
"owner_lane": "AutomationRouter",
"risk": "high",
"status": "source_implemented_runtime_pending",
"dependencies": [
"AIA-SRE-003"
],
"source_refs": [
"apps/api/src/services/controlled_alert_target_router.py",
"apps/api/src/services/awooop_ansible_audit_service.py",
"apps/api/src/services/awooop_ansible_check_mode_service.py"
],
"executor": "typed policy router",
"verifier": "domain/catalog/host/canonical ID scope verifier",
"rollback": "revert router source while keeping unknown fail-closed",
"exit_condition": "every candidate is exact-domain scoped and circuit-open never selects another host/domain",
"next_action": "run full Ansible replay regression"
},
{
"id": "AIA-SRE-005",
"order": 5,
"priority": "P0",
"phase": "phase_1",
"title": "Sentry profiling consumer exact-container bounded recovery",
"owner_lane": "HostContainerAutomation",
"risk": "medium",
"status": "source_implemented_runtime_pending",
"dependencies": [
"AIA-SRE-004"
],
"source_refs": [
"infra/ansible/playbooks/110-sentry-profiling-consumer-recovery.yml",
"apps/api/src/services/awooop_ansible_post_verifier.py"
],
"executor": "awoooi-ansible-executor-broker",
"verifier": "exact container running plus healthy readback",
"rollback": "stop further writes, same-domain check replay and cooldown",
"exit_condition": "INC-20260714-471307 replay selects only host_110 exact playbook and post-verifier passes",
"next_action": "deploy then execute one bounded same-fingerprint controlled replay"
},
{
"id": "AIA-SRE-006",
"order": 6,
"priority": "P0",
"phase": "phase_2",
"title": "Agent99 Host Operations Bridge typed enforcement",
"owner_lane": "Agent99",
"risk": "high",
"status": "source_implemented_runtime_pending",
"dependencies": [
"AIA-SRE-004"
],
"source_refs": [
"apps/api/src/services/agent99_sre_bridge.py",
"apps/api/src/services/agent99_controlled_dispatch_ledger.py",
"apps/api/src/services/agent99_completion_callback.py"
],
"executor": "Agent99 for Windows/VMware/control-plane; relay-only elsewhere",
"verifier": "Agent99 callback plus external same-run verifier",
"rollback": "disable typed dispatch route and retain no-write Status readback",
"exit_condition": "all Agent99 dispatches carry canonical typed scope and no legacy catch-all can mutate",
"next_action": "deploy then replay one allowlisted Host112 or cold-start run plus one cross-domain denied candidate"
},
{
"id": "AIA-SRE-007",
"order": 7,
"priority": "P0",
"phase": "phase_2",
"title": "K3s workload 專屬 executor 與 rollout verifier",
"owner_lane": "KubernetesAutomation",
"risk": "high",
"status": "planned",
"dependencies": [
"AIA-SRE-004"
],
"source_refs": [
"apps/api/src/services/executor.py",
"apps/api/src/services/post_execution_verifier.py"
],
"executor": "kubernetes_controlled_executor",
"verifier": "kubernetes_rollout_verifier",
"rollback": "declarative rollout undo or previous immutable revision",
"exit_condition": "K3s incidents never enter Ansible/Agent99 fallback and close with rollout evidence",
"next_action": "inventory current kubernetes actions and bind allowlisted declarative routes"
},
{
"id": "AIA-SRE-008",
"order": 8,
"priority": "P0",
"phase": "phase_2",
"title": "Host/systemd 專屬 Ansible executor inventory",
"owner_lane": "HostAutomation",
"risk": "high",
"status": "in_progress",
"dependencies": [
"AIA-SRE-004"
],
"source_refs": [
"infra/ansible/inventory/hosts.yml",
"apps/api/src/services/awooop_ansible_audit_service.py"
],
"executor": "host_ansible_executor",
"verifier": "host_runtime_independent_verifier",
"rollback": "catalog-specific compensating action",
"exit_condition": "every host action has one exact inventory host, check mode, bounded apply and postcondition",
"next_action": "close catalogs still backed by broad convergence playbooks"
},
{
"id": "AIA-SRE-009",
"order": 9,
"priority": "P0",
"phase": "phase_2",
"title": "Windows/VMware Agent99 single-executor closure",
"owner_lane": "Agent99",
"risk": "high",
"status": "in_progress",
"dependencies": [
"AIA-SRE-006"
],
"source_refs": [
"apps/api/src/services/agent99_controlled_dispatch_ledger.py",
"apps/api/src/services/agent99_same_run_reconcile.py"
],
"executor": "Agent99",
"verifier": "agent99_independent_runtime_verifier",
"rollback": "allowlisted Status/no-write reconciliation and bounded generation retry",
"exit_condition": "Windows/VMware runs use durable identity, callback, verifier and learning receipt",
"next_action": "reconcile remaining Agent99 enterprise work ledger with this program"
},
{
"id": "AIA-SRE-010",
"order": 10,
"priority": "P0",
"phase": "phase_3",
"title": "DB 專屬 verifier 與 bounded executor",
"owner_lane": "DatabaseAutomation",
"risk": "high",
"status": "planned",
"dependencies": [
"AIA-SRE-004"
],
"source_refs": [
"ops/config/service-registry.yaml"
],
"executor": "db_bounded_executor",
"verifier": "db_independent_verifier",
"rollback": "transactional or compensating action only; destructive restore remains break-glass",
"exit_condition": "DB incidents use DB-native preflight/postconditions and never generic host restart",
"next_action": "classify read-only, session, pool, replication and bounded maintenance actions"
},
{
"id": "AIA-SRE-011",
"order": 11,
"priority": "P0",
"phase": "phase_3",
"title": "Backup/restore readback 與 critical break-glass",
"owner_lane": "DataProtection",
"risk": "critical",
"status": "policy_active",
"dependencies": [
"AIA-SRE-004"
],
"source_refs": [
"apps/api/src/services/backup_restore_signal_automation.py"
],
"executor": "backup_restore_break_glass",
"verifier": "backup_restore_readback_verifier",
"rollback": "no-write default; destructive action requires incident-specific break-glass",
"exit_condition": "freshness, offsite, escrow and restore-drill evidence close without false green",
"next_action": "bind typed route to DR scorecard without running backup/restore"
},
{
"id": "AIA-SRE-012",
"order": 12,
"priority": "P0",
"phase": "phase_3",
"title": "HolmesGPT Investigator shadow integration",
"owner_lane": "Investigator",
"risk": "high",
"status": "planned",
"dependencies": [
"AIA-SRE-004"
],
"source_refs": [
"apps/api/src/services/pre_decision_investigator.py"
],
"executor": "none in shadow",
"verifier": "offline replay quality and prompt-injection safety scorecard",
"rollback": "remove shadow consumer; active PreDecisionInvestigator remains",
"exit_condition": "internal immutable artifact passes replay, shadow and canary before core replacement",
"next_action": "source HolmesGPT from approved internal mirror without GitHub access"
},
{
"id": "AIA-SRE-013",
"order": 13,
"priority": "P0",
"phase": "phase_3",
"title": "Ollama RCA 與 Gemini Critic/cost gate",
"owner_lane": "ModelRouter",
"risk": "critical",
"status": "in_progress",
"dependencies": [
"AIA-SRE-012"
],
"source_refs": [
"apps/api/src/services/ai_router.py",
"apps/api/src/services/ollama_failover.py"
],
"executor": "ordered model router",
"verifier": "provider-attributed replay accuracy, latency and cost readback",
"rollback": "Gemini disabled; remain on ordered Ollama chain",
"exit_condition": "provider order is enforced and Gemini cannot run without explicit cost cap",
"next_action": "verify secret metadata presence without reading value, then build offline comparison"
},
{
"id": "AIA-SRE-014",
"order": 14,
"priority": "P0",
"phase": "phase_4",
"title": "Independent verifier registry 完整覆蓋",
"owner_lane": "PostExecutionVerifier",
"risk": "high",
"status": "in_progress",
"dependencies": [
"AIA-SRE-005",
"AIA-SRE-007",
"AIA-SRE-008",
"AIA-SRE-009",
"AIA-SRE-010"
],
"source_refs": [
"apps/api/src/services/awooop_ansible_post_verifier.py",
"apps/api/src/services/post_execution_verifier.py"
],
"executor": "none",
"verifier": "domain-specific external readback",
"rollback": "failed verifier triggers same-domain retry or compensating action",
"exit_condition": "100 percent of mutating catalogs have independent postconditions",
"next_action": "reconcile catalog IDs against postcondition registry"
},
{
"id": "AIA-SRE-015",
"order": 15,
"priority": "P0",
"phase": "phase_4",
"title": "Incident closure 與 KM/RAG/MCP/PlayBook durable writeback",
"owner_lane": "LearningControlPlane",
"risk": "medium",
"status": "in_progress",
"dependencies": [
"AIA-SRE-014"
],
"source_refs": [
"apps/api/src/services/learning_service.py",
"apps/api/src/services/awooop_ansible_learning_writeback.py"
],
"executor": "learning writeback consumer",
"verifier": "durable same-run acknowledgement readback",
"rollback": "append-only correction event; immutable receipts are not overwritten",
"exit_condition": "closure requires all learning acknowledgements on the same run",
"next_action": "bind typed domain and verifier outcome into immutable learning receipt"
},
{
"id": "AIA-SRE-016",
"order": 16,
"priority": "P1",
"phase": "phase_5",
"title": "24h replay、MTTA/MTTR、誤判、復發與成本 scorecard",
"owner_lane": "AutomationSLO",
"risk": "low",
"status": "planned",
"dependencies": [
"AIA-SRE-015"
],
"source_refs": [],
"executor": "offline replay worker",
"verifier": "production aggregate versus same-run receipts",
"rollback": "read-only scorecard",
"exit_condition": "all lanes publish MTTA, MTTR, false positive, recurrence, human intervention, verifier and rollback metrics",
"next_action": "replay the latest 24h alert/runs corpus after phase 1 deploy"
},
{
"id": "AIA-SRE-017",
"order": 17,
"priority": "P1",
"phase": "phase_5",
"title": "Telegram 與 cockpit 只呈現 canonical lifecycle truth",
"owner_lane": "ProductExperience",
"risk": "medium",
"status": "in_progress",
"dependencies": [
"AIA-SRE-015"
],
"source_refs": [
"ops/config/telegram-routing-registry.yaml"
],
"executor": "canonical Telegram gateway",
"verifier": "delivery receipt plus desktop/mobile visible smoke",
"rollback": "suppress duplicate projection; retain incident ledger",
"exit_condition": "no alert is sprayed across groups/bots and every card links one canonical run",
"next_action": "reconcile project/product/alert class routing against the typed domain"
},
{
"id": "AIA-SRE-018",
"order": 18,
"priority": "P0",
"phase": "phase_6",
"title": "Gitea CD、production runtime 與 visible closure",
"owner_lane": "ReleaseControl",
"risk": "high",
"status": "planned",
"dependencies": [
"AIA-SRE-001",
"AIA-SRE-002",
"AIA-SRE-003",
"AIA-SRE-004",
"AIA-SRE-005",
"AIA-SRE-006",
"AIA-SRE-014",
"AIA-SRE-015"
],
"source_refs": [
".gitea/workflows/cd.yaml"
],
"executor": "Gitea controlled CD",
"verifier": "deploy marker, image SHA, API readback and desktop/mobile smoke",
"rollback": "previous immutable image revision and ConfigMap source SHA",
"exit_condition": "source/test/CD/runtime/UI evidence all match one deployed SHA",
"next_action": "commit phase 1, normal push after shared writer release, then terminal CD/readback"
}
],
"rollups": {
"total_items": 18,
"by_priority": {
"P0": 16,
"P1": 2
},
"by_status": {
"source_implemented_runtime_pending": 6,
"in_progress": 6,
"planned": 5,
"policy_active": 1
},
"source_implemented_items": 6,
"runtime_closed_items": 0,
"program_completion_percent": 0,
"asset_coverage_status": "partial",
"runtime_closure_status": "not_started_for_this_program_sha"
},
"completion_contract": {
"required_stages": [
"sensor_source_receipt",
"canonical_asset_identity",
"source_truth_diff",
"ai_decision_candidate",
"risk_policy_decision",
"check_mode_receipt",
"bounded_execution_receipt",
"independent_verifier_or_no_write_terminal",
"incident_closure",
"telegram_receipt",
"km_rag_mcp_playbook_writeback_ack"
],
"terminal_without_all_stages": "partial_or_degraded_with_safe_next_action",
"source_test_cd_green_is_runtime_closure": false
}
}