feat(agent99): establish enterprise AI automation work ledger

This commit is contained in:
ogt
2026-07-10 15:20:23 +08:00
parent a8080e13b9
commit 579d12c3fa
10 changed files with 1283 additions and 2 deletions

View File

@@ -0,0 +1,747 @@
{
"schema_version": "agent99_enterprise_ai_automation_work_items_v1",
"generated_at": "2026-07-10T15:05:00+08:00",
"status": "in_progress_p0_agent_outcome_truth",
"scope_complete": false,
"north_star": "Make Windows host 192.168.0.99 the policy-controlled on-prem AI execution node for all hosts, VMs, services, products, sites, tools, packages, alerts and recovery workflows, with external monitoring when host 99 is unavailable.",
"canonical_sources": {
"human_plan": "docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md",
"historical_requirement_ledger": "docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md",
"agent99_sop": "AGENT99-AI-WORKSTATION-SOP.md",
"reboot_sop": "HOST-REBOOT-AI-AUTOMATION-SOP.md",
"cold_start_baseline": "ops/reboot-recovery/full-stack-cold-start-baseline.yml",
"backup_baseline": "ops/reboot-recovery/full-stack-backup-baseline.yml",
"product_repo_inventory": "docs/operations/gitea-all-product-dev-prod-repo-readback.snapshot.json",
"runtime_enterprise_work_items_readback": "https://awoooi.wooo.work/api/v1/agents/agent99-enterprise-ai-automation-work-items",
"runtime_priority_readback": "https://awoooi.wooo.work/api/v1/agents/awoooi-priority-work-order-readback",
"runtime_reboot_slo_readback": "https://awoooi.wooo.work/api/v1/agents/reboot-auto-recovery-slo-scorecard"
},
"current_p0": {
"id": "AG99-P0-001",
"title": "Agent outcome truth and source-runtime reconciliation",
"reason": "Agent99 currently treats command exit success as incident resolution and the Windows runtime scripts do not consistently match the current Gitea source. All later automation depends on trustworthy outcomes.",
"next_action": "Implement mode-specific outcome evaluation, require verifier and source-event resolution for resolved, add regression tests, then deploy and compare source/runtime hashes on host 99."
},
"current_runtime_truth": {
"observed_at": "2026-07-10T15:04:27+08:00",
"awoooi_health_http_status": 200,
"awoooi_health_state": "degraded",
"reboot_slo_status": "blocked_reboot_auto_recovery_slo_not_ready",
"reboot_slo_readiness_percent": 60,
"reboot_slo_primary_blocker": "reboot_event_required_host_unreachable",
"reboot_slo_active_blockers": [
"all_required_hosts_not_in_10_minute_reboot_window",
"fresh_all_host_reboot_event_missing",
"host_boot_observation_older_than_target_window",
"host_unreachable_after_reboot",
"host_uptime_unknown",
"reboot_event_required_host_unreachable",
"windows99_vmware_guest_power_not_ready",
"edge_public_maintenance_live_config_drift",
"windows99_vmware_vmx_missing",
"windows99_update_no_auto_reboot_policy_not_ready",
"privileged_edge_apply_channel_unavailable",
"windows99_vmware_autostart_config_not_ready"
],
"claims_forbidden": [
"all_services_recovered_within_10_minutes",
"all_products_are_latest",
"agent99_is_a_complete_ai_agent",
"http_200_means_full_stack_healthy",
"command_exit_zero_means_incident_resolved"
]
},
"coverage": {
"hosts": [
{
"id": "99",
"address": "192.168.0.99",
"role": "Windows AI execution and VMware control node",
"required_surfaces": ["Agent99", "VMware Workstation", "vmrun", "Task Scheduler", "PowerShell", "OpenSSH", "Windows Update policy", "desktop control center"]
},
{
"id": "110",
"address": "192.168.0.110",
"role": "DevOps, SCM, registry, observability and backup node",
"required_surfaces": ["Gitea", "Gitea Actions runner", "Docker", "Harbor", "Registry", "Prometheus", "Alertmanager", "Sentry", "backup"]
},
{
"id": "111",
"address": "192.168.0.111",
"role": "Required VMware guest with role manifest pending",
"required_surfaces": ["verified VMX inventory", "VM power", "boot identity", "uptime", "SSH", "exporter", "service role manifest"]
},
{
"id": "112",
"address": "192.168.0.112",
"role": "Kali and IWOOOS security node",
"required_surfaces": ["VM power", "boot identity", "SSH", "security tool health", "IWOOOS evidence collector"]
},
{
"id": "120",
"address": "192.168.0.120",
"role": "K3s control plane, VIP and AWOOOI workload node",
"required_surfaces": ["K3s", "containerd", "kubectl", "VIP", "AWOOOI deployments", "Velero", "CronJobs"]
},
{
"id": "121",
"address": "192.168.0.121",
"role": "K3s peer and DR drill node",
"required_surfaces": ["K3s", "containerd", "peer readiness", "workload balance", "Velero restore drill"]
},
{
"id": "188",
"address": "192.168.0.188",
"role": "Data, AI, observability and web workload node",
"required_surfaces": ["PostgreSQL", "Redis", "MOMO", "Nginx", "OpenClaw", "Ollama", "SignOz", "Langfuse", "AI providers", "backup"]
}
],
"products": [
{"id": "awoooi", "gitea_repo": "wooo/awoooi"},
{"id": "ewoooc", "gitea_repo": "wooo/ewoooc"},
{"id": "2026fifa", "gitea_repo": "wooo/2026FIFAWorldCup"},
{"id": "agent-bounty-protocol", "gitea_repo": "wooo/agent-bounty-protocol"},
{"id": "awooogo", "gitea_repo": "wooo/AwoooGo"},
{"id": "stockplatform-v2", "gitea_repo": "wooo/stockplatform-v2"},
{"id": "vibework", "gitea_repo": "wooo/vibework"},
{"id": "momo-pro-system", "gitea_repo": "wooo/momo-pro-system"},
{"id": "tsenyang-website", "gitea_repo": "wooo/tsenyang-website"},
{"id": "vtuber", "gitea_repo": "wooo/vtuber"},
{"id": "bitan-pharmacy", "gitea_repo": "wooo/bitan-pharmacy"},
{"id": "clawbot-openclaw", "gitea_repo": "wooo/clawbot-v5"}
],
"platform_planes": ["AWOOOI", "AwoooP", "IWOOOS", "AISOC", "AgentOps", "OpenClaw", "KM", "RAG", "MCP", "PlayBook"],
"public_surfaces": [
{"id": "awoooi-api", "url": "https://awoooi.wooo.work/api/v1/health", "owner": "awoooi"},
{"id": "awoooi-web", "url": "https://awoooi.wooo.work/", "owner": "awoooi"},
{"id": "iwooos", "url": "https://awoooi.wooo.work/zh-TW/iwooos", "owner": "awoooi"},
{"id": "vibework", "url": "https://vibework.wooo.work/", "owner": "vibework"},
{"id": "awooogo", "url": "https://awooogo.wooo.work/", "owner": "awooogo"},
{"id": "2026fifa", "url": "https://2026fifa.wooo.work/", "owner": "2026fifa"},
{"id": "agent", "url": "https://agent.wooo.work/", "owner": "ownership_contract_pending"},
{"id": "momo", "url": "https://mo.wooo.work/", "owner": "momo-pro-system"},
{"id": "momo-health", "url": "https://mo.wooo.work/health", "owner": "momo-pro-system"},
{"id": "stock", "url": "https://stock.wooo.work/", "owner": "stockplatform-v2"},
{"id": "stock-health", "url": "https://stock.wooo.work/healthz", "owner": "stockplatform-v2"},
{"id": "stock-api", "url": "https://stock.wooo.work/api/healthz", "owner": "stockplatform-v2"},
{"id": "bitan", "url": "https://bitan.wooo.work/", "owner": "bitan-pharmacy"},
{"id": "tsenyang", "url": "https://tsenyang.com/", "owner": "tsenyang-website"},
{"id": "tsenyang-www", "url": "https://www.tsenyang.com/", "owner": "tsenyang-website"},
{"id": "vtuber", "url": "https://vtuber.wooo.work/", "owner": "vtuber"},
{"id": "gitea", "url": "https://gitea.wooo.work/", "owner": "platform-110"},
{"id": "harbor", "url": "https://harbor.wooo.work/", "owner": "platform-110"},
{"id": "registry", "url": "https://registry.wooo.work/", "owner": "platform-110"},
{"id": "sentry", "url": "https://sentry.wooo.work/", "owner": "platform-observability"},
{"id": "signoz", "url": "https://signoz.wooo.work/", "owner": "platform-observability"},
{"id": "langfuse", "url": "https://langfuse.wooo.work/", "owner": "platform-ai"},
{"id": "aiops", "url": "https://aiops.wooo.work/", "owner": "ownership_contract_pending"}
],
"service_domains": [
"windows_and_vmware",
"network_and_dns",
"linux_systemd_and_schedulers",
"docker_containerd_and_k3s",
"gitea_runner_harbor_registry_and_cd",
"postgresql_redis_and_object_data",
"nginx_tls_edge_and_maintenance",
"prometheus_alertmanager_sentry_signoz_langfuse_and_wazuh",
"openclaw_hermes_nemotron_elephantalpha_ollama_litellm_and_models",
"backup_restic_rclone_velero_offsite_integrity_and_restore",
"km_rag_mcp_playbook_and_agentops",
"telegram_sre_and_operator_command_surfaces"
],
"package_domains": [
"windows_updates_and_drivers",
"linux_os_packages_and_kernels",
"container_images_and_digests",
"python_node_and_application_dependencies",
"vmware_tools_and_guest_agents",
"ai_models_adapters_and_provider_clients",
"security_rules_signatures_and_sbom"
]
},
"control_plane": [
{"node": "external_watcher_edge", "responsibility": "Detect host 99 or all-site outage and keep alerts/status/maintenance available outside the local failure domain."},
{"node": "awoooi", "responsibility": "Canonical events, incidents, fingerprints, recurrence, product and SLO read models."},
{"node": "awooop", "responsibility": "Work Items, Runs, approvals, candidates, rollback, verifier and asset ledger."},
{"node": "iwooos_aisoc", "responsibility": "Security evidence, policy, investigation and controlled response."},
{"node": "agent99", "responsibility": "Evidence collection, planning, policy-bound execution, verification and callbacks for Windows, VMs, hosts, services and sites."},
{"node": "km_rag_mcp_playbook", "responsibility": "Replayable evidence, knowledge, tools, trusted remediation and learning."},
{"node": "telegram_sre", "responsibility": "Human-readable command and incident summary surface, never the only database or an arbitrary shell."}
],
"automation_flow": [
"sense",
"normalize",
"group_and_dedupe",
"collect_evidence",
"plan",
"policy_decision",
"controlled_execute",
"mode_specific_verify",
"callback_and_notify",
"learn_and_prevent_recurrence"
],
"outcome_contract": {
"states": ["received", "diagnosing", "candidate_ready", "executing", "verifying", "resolved", "degraded", "blocked", "failed"],
"resolved_requires": ["verifier_passed", "source_event_resolved", "evidence_ref", "verified_at", "callback_written"],
"exit_code_zero_semantics": "command_transport_completed_only",
"observation_window_required_after_resolved": true
},
"professional_controls": [
"external_failure_domain_monitoring",
"zero_trust_authenticated_ingress",
"event_sourcing_and_idempotency",
"single_flight_and_concurrency_locks",
"dependency_dag_and_recovery_state_machine",
"mode_specific_verifiers",
"progressive_remediation_and_canary",
"circuit_breakers_timeouts_and_real_rollback",
"immutable_redacted_evidence",
"source_cd_runtime_and_data_freshness_provenance",
"slo_error_budget_mtta_mttr_and_recurrence",
"playbook_replay_shadow_canary_and_trust_promotion",
"sbom_cve_eol_package_and_model_drift",
"backup_integrity_offsite_and_restore_drills",
"human_readable_traditional_chinese_incident_cards"
],
"hard_boundaries": [
"secret_token_private_key_cookie_session_or_authorization_header_plaintext_access_or_exfiltration",
"drop_truncate_destructive_migration_restore_prune_or_remote_delete",
"host_reboot_vm_power_change_node_drain_or_irreversible_firewall_cutover",
"credentialed_exploit_or_external_attack_scan",
"paid_provider_cost_change_or_production_provider_route_switch",
"unvalidated_openclaw_core_replacement",
"force_push_repo_ref_visibility_or_source_history_destruction",
"raw_runtime_secret_volume_session_sqlite_or_auth_database_access"
],
"shared_acceptance_contract": [
"target_selector_present",
"source_of_truth_diff_present",
"before_evidence_present",
"check_mode_or_dry_run_present",
"rollback_present",
"mode_specific_post_verifier_present",
"timeout_and_single_flight_present",
"callback_and_human_readable_notification_present",
"km_rag_playbook_and_recurrence_writeback_present"
],
"promotion_contract": {
"applies_before_states": ["candidate_ready", "in_progress", "executing"],
"required_item_fields": [
"id",
"order",
"priority",
"status",
"title",
"owner_lane",
"dependencies",
"scope_refs",
"problem",
"trigger",
"ai_role",
"action_mode",
"evidence_sources",
"controlled_actions",
"prohibited_actions",
"rollback",
"verifier",
"callback",
"acceptance",
"next_action"
]
},
"work_items": [
{
"id": "AG99-P0-001",
"order": 1,
"priority": "P0",
"status": "in_progress",
"title": "Agent outcome truth and source-runtime reconciliation",
"owner_lane": "agent99_control_plane",
"dependencies": [],
"scope_refs": ["host:99", "platform:Agent99", "platform:AWOOOI", "platform:AwoooP"],
"problem": "Command exit success is counted as resolved and deployed Windows scripts can drift from Gitea source.",
"trigger": "Any Agent99 command completion or source/runtime manifest mismatch.",
"ai_role": "Outcome evaluator and runtime provenance verifier",
"action_mode": "controlled_apply",
"evidence_sources": ["Agent99 event store", "mode evidence JSON", "source file hashes", "host 99 runtime file hashes", "AWOOOI incident state"],
"controlled_actions": ["replace outcome mapping", "add verifier contract", "deploy signed script set", "write runtime manifest"],
"prohibited_actions": ["mark resolved from exit code alone", "read secrets", "reboot host 99"],
"rollback": "Restore previous signed Agent99 script set and keep incidents degraded rather than resolved.",
"verifier": "For every mode, require post-condition pass plus source event resolution; compare source and runtime manifest hashes.",
"callback": "Write final outcome to AWOOOI incident and AwoooP Run/Verifier using the same trace id.",
"acceptance": "No false-resolved regression cases; source/runtime drift count is zero; unresolved evidence stays degraded or blocked.",
"next_action": "Patch agent99-control-plane.ps1 outcome evaluation and add focused regression tests."
},
{
"id": "AG99-P0-002",
"order": 2,
"priority": "P0",
"status": "planned",
"title": "Alert taxonomy, routing, grouping and dedupe",
"owner_lane": "awoooi_alert_router_agent99_ingress",
"dependencies": ["AG99-P0-001"],
"scope_refs": ["all:alerts", "platform:AWOOOI", "host:99"],
"problem": "Source text contaminates routing and the bridge forwards before grouping, creating wrong modes and duplicate incidents.",
"trigger": "Any normalized alert or Telegram monitoring event entering the AWOOOI-to-Agent99 bridge.",
"ai_role": "Normalized incident classifier",
"action_mode": "controlled_apply",
"evidence_sources": ["Alertmanager payload", "suggestedMode", "event kind", "fingerprint", "five-minute grouping window"],
"controlled_actions": ["prioritize explicit kind and suggested mode", "group before bridge", "single-flight by incident fingerprint"],
"prohibited_actions": ["route from source name alone", "emit duplicate same-state incidents"],
"rollback": "Restore previous router while keeping dedupe store and queued events intact.",
"verifier": "Replay routing corpus and assert one active incident and expected mode per fingerprint.",
"callback": "Publish normalized route decision and dedupe reason to AWOOOI and AwoooP.",
"acceptance": "Known backup, provider, reboot, PostgreSQL, host and K3s samples route to the correct lane with no duplicate same-state dispatch.",
"next_action": "Add routing precedence and pre-bridge grouping regression corpus."
},
{
"id": "AG99-P0-003",
"order": 3,
"priority": "P0",
"status": "planned",
"title": "Authenticated and policy-preserving Agent99 ingress",
"owner_lane": "agent99_security",
"dependencies": ["AG99-P0-002"],
"scope_refs": ["host:99", "surface:Telegram", "surface:LAN relay", "platform:IWOOOS"],
"problem": "Relay authentication is optional, the LAN is broadly trusted, producer repair policy can be overwritten and Telegram lacks a sender allowlist.",
"trigger": "Any LAN relay, Telegram command or monitoring-event ingress request.",
"ai_role": "Zero-trust command gate",
"action_mode": "controlled_apply",
"evidence_sources": ["relay auth metadata", "Telegram chat and sender metadata", "request policy", "idempotency key"],
"controlled_actions": ["require relay token presence without exposing it", "enforce chat and sender allowlists", "preserve producer auto-repair policy", "single consumer lease"],
"prohibited_actions": ["accept unauthenticated LAN writes", "log token or raw chat id", "force controlled apply at producer"],
"rollback": "Disable remote ingress and retain local signed queue until authentication is healthy.",
"verifier": "Negative auth tests, replay protection, allowlist tests and policy preservation tests.",
"callback": "Record accepted or rejected ingress receipt without secret values.",
"acceptance": "Unauthenticated, replayed or unauthorized requests are rejected; authorized requests preserve source policy and trace identity.",
"next_action": "Make relay authentication mandatory and add Telegram sender authorization."
},
{
"id": "AG99-P0-004",
"order": 4,
"priority": "P0",
"status": "planned",
"title": "EvidenceCollect, MCP and RepairCandidate lanes",
"owner_lane": "agent99_investigator_awooop_candidate",
"dependencies": ["AG99-P0-001", "AG99-P0-002", "AG99-P0-003"],
"scope_refs": ["all:incidents", "platform:MCP", "platform:AwoooP"],
"problem": "Events with missing evidence are sent to unrelated fixed modes and repair candidates are not materialized or traceable.",
"trigger": "An actionable incident lacks a trusted PlayBook, complete evidence or a safe executable candidate.",
"ai_role": "Evidence investigator and controlled remediation planner",
"action_mode": "observe_then_controlled_apply",
"evidence_sources": ["metrics", "logs", "runtime state", "recent changes", "source and deploy provenance", "MCP references"],
"controlled_actions": ["collect redacted evidence", "create candidate", "attach repair rollback and verifier", "queue owner review when required"],
"prohibited_actions": ["execute generic fallback repair", "invent missing evidence", "switch provider"],
"rollback": "Expire the candidate without runtime action and preserve the evidence packet.",
"verifier": "Incident-to-candidate trace and required-field completeness checker.",
"callback": "Create or update AwoooP Work Item and AWOOOI incident candidate status.",
"acceptance": "Every actionable missing-evidence incident has a candidate or an explicit blocked reason, not a false resolved result.",
"next_action": "Add first-class EvidenceCollect and RepairCandidate modes and state directories."
},
{
"id": "AG99-P0-005",
"order": 5,
"priority": "P0",
"status": "planned",
"title": "Mode-specific verifiers, rollback, cooldown and circuit breakers",
"owner_lane": "agent99_remediation_runtime",
"dependencies": ["AG99-P0-004"],
"scope_refs": ["all:remediation_modes"],
"problem": "Generic execution success and permissive verifier logic can write cooldown markers after failed repairs; rollback is often only recorded.",
"trigger": "Any remediation mode reaches execution, verification, rollback or cooldown evaluation.",
"ai_role": "Remediation executor and post-condition verifier",
"action_mode": "controlled_apply",
"evidence_sources": ["before state", "command receipt", "post state", "source alert", "observation window"],
"controlled_actions": ["run allowlisted repair", "perform actual rollback", "open circuit on repeated failures", "write cooldown only after verified success"],
"prohibited_actions": ["generic stateful restart", "cooldown on failed verify", "delete healthy pods"],
"rollback": "Mode-specific executable rollback with its own verifier.",
"verifier": "Recover, Perf, LoadShed, Harbor, AWOOOI, backup and provider modes each define independent post-conditions.",
"callback": "Report execution and verification separately to AWOOOI and AwoooP.",
"acceptance": "Failure, rollback and resolved states are distinguishable and regression-tested for every executable mode.",
"next_action": "Replace OR-based load-shed success and add verifier interfaces per mode."
},
{
"id": "AG99-P0-006",
"order": 6,
"priority": "P0",
"status": "planned",
"title": "Agent99 completion callback and incident closure",
"owner_lane": "agent99_awoooi_awooop_integration",
"dependencies": ["AG99-P0-001", "AG99-P0-005"],
"scope_refs": ["platform:AWOOOI", "platform:AwoooP", "platform:IWOOOS", "surface:Telegram"],
"problem": "AWOOOI-to-Agent99 dispatch is not matched by a canonical Agent99-to-AWOOOI completion path.",
"trigger": "Any Agent99 run changes state or produces a verifier result.",
"ai_role": "Cross-system incident closer",
"action_mode": "controlled_apply",
"evidence_sources": ["incident id", "run id", "work item id", "verifier receipt", "recurrence state"],
"controlled_actions": ["write signed callback", "update Run and Work Item", "render human notification", "reopen on recurrence"],
"prohibited_actions": ["close by Telegram send success", "create unrelated incident ids"],
"rollback": "Queue callback retry idempotently without repeating the repair.",
"verifier": "Read back identical trace, final state and verifier evidence from AWOOOI and AwoooP.",
"callback": "This work item creates the canonical callback contract.",
"acceptance": "One trace is visible from alert receipt through final verifier, notification and recurrence counters.",
"next_action": "Define callback schema and idempotent authenticated endpoint."
},
{
"id": "AG99-P0-007",
"order": 7,
"priority": "P0",
"status": "in_progress_blocked",
"title": "All-host ten-minute reboot recovery state machine",
"owner_lane": "reboot_auto_recovery",
"dependencies": ["AG99-P0-001", "AG99-P0-005", "AG99-P0-006"],
"scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188", "all:public_surfaces"],
"problem": "Reboot detection and recovery remain one-shot or incomplete, with stale uptime, unreachable host and VM power blockers.",
"trigger": "Fresh boot identity, host down/up transition, Windows startup task or all-host reboot correlation.",
"ai_role": "Dependency-aware reboot recovery orchestrator",
"action_mode": "controlled_apply",
"evidence_sources": ["boot id", "uptime", "VM power", "SSH", "exporters", "service DAG", "public and data freshness"],
"controlled_actions": ["detect reboot epoch", "start dependency phases", "retry bounded stateless recovery", "emit ETA and blockers"],
"prohibited_actions": ["blind host reboot", "declare green from route 200", "skip data and backup checks"],
"rollback": "Stop at last verified phase, activate maintenance state and preserve dependent workloads.",
"verifier": "Measure detection time, first availability, full green time and every required host/service/product gate.",
"callback": "Write reboot incident phase and SLO result to AWOOOI, AwoooP and Telegram.",
"acceptance": "A fresh all-host reboot drill is detected and all required gates recover within 600 seconds or report a precise blocker and ETA.",
"next_action": "After AG99-P0-001 through AG99-P0-004, rerun no-secret collector and state-machine scorecard without VM power changes."
},
{
"id": "AG99-P0-008",
"order": 8,
"priority": "P0",
"status": "in_progress_blocked",
"title": "Windows 99 VMware autostart, VMX, lock and update policy",
"owner_lane": "windows99_vmware",
"dependencies": ["AG99-P0-003", "AG99-P0-005"],
"scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188"],
"problem": "Verified VMX inventory, guest power readback, VMware autostart and Windows no-auto-reboot policy are not all green.",
"trigger": "Windows startup, VMware exclusive-lock error, guest power mismatch, VMX drift or Windows Update policy drift.",
"ai_role": "Windows and VMware control agent",
"action_mode": "check_mode_then_controlled_apply",
"evidence_sources": ["VMX inventory", "vmrun list", "VMware services", "scheduled tasks", "lock owners", "Windows Update policy"],
"controlled_actions": ["repair stale lock after owner proof", "configure autostart order", "verify guest tools and network", "apply reversible update policy"],
"prohibited_actions": ["delete unknown lock", "guess VMX path", "power off VM", "reboot Windows without break-glass"],
"rollback": "Restore previous task, service and policy exports; keep guest power unchanged during check mode.",
"verifier": "All required VMX paths exist, guests report running and reachable, tasks survive reboot, update policy drift is zero.",
"callback": "Update reboot SLO VMware and Windows policy gates.",
"acceptance": "99 startup automatically launches VMware and required guests in order, with no exclusive-lock failure and no surprise Windows Update reboot.",
"next_action": "Complete 111 role/VMX inventory and build the guest-power readback check mode."
},
{
"id": "AG99-P0-009",
"order": 9,
"priority": "P0",
"status": "in_progress_blocked",
"title": "External outage watcher and maintenance failover",
"owner_lane": "edge_external_observability",
"dependencies": ["AG99-P0-002", "AG99-P0-006"],
"scope_refs": ["host:99", "all:hosts", "all:public_surfaces"],
"problem": "Host 99 cannot detect its own outage and edge fallback has live configuration drift, allowing raw 502 during local recovery.",
"trigger": "External synthetic failure, local recovery incident or raw public 5xx threshold breach.",
"ai_role": "External availability observer and maintenance-state controller",
"action_mode": "controlled_apply",
"evidence_sources": ["external synthetic probes", "edge health", "maintenance headers", "local recovery state"],
"controlled_actions": ["send down and up incidents", "activate pre-provisioned maintenance origin", "withdraw fallback after verifier"],
"prohibited_actions": ["daily DNS mutation", "hide partial outage", "depend on 99 for all-host-down detection"],
"rollback": "Restore previous edge origin and keep status page incident open until public verifier passes.",
"verifier": "External probe sees maintenance content instead of raw 502 during outage and normal routes after recovery.",
"callback": "Correlate external outage with reboot incident and Telegram lifecycle card.",
"acceptance": "Down, recovering and recovered remain observable outside the LAN, and public users never receive unmanaged 502 for a known full-stack recovery.",
"next_action": "Resolve edge live-config drift and define an independent watcher runtime outside 99/110/188."
},
{
"id": "AG99-P0-010",
"order": 10,
"priority": "P0",
"status": "in_progress",
"title": "All-product version and data freshness matrix",
"owner_lane": "product_runtime_governance",
"dependencies": ["AG99-P0-006"],
"scope_refs": ["all:products", "all:public_surfaces", "all:data_sources"],
"problem": "Repository presence and HTTP 200 do not prove that every product runtime and dataset is current.",
"trigger": "Source change, CD completion, scheduled freshness audit or product stale/degraded alert.",
"ai_role": "Release and freshness auditor",
"action_mode": "observe_then_controlled_apply",
"evidence_sources": ["Gitea main and dev SHA", "Gitea CD run", "deploy marker", "runtime image digest", "public smoke", "data last seen"],
"controlled_actions": ["flag drift", "queue product-specific deploy candidate", "rerun freshness ingestion verifier"],
"prohibited_actions": ["claim latest from repo alone", "force deploy without product verifier", "switch paid provider"],
"rollback": "Use product-specific previous image or release marker and rerun functional/data verifiers.",
"verifier": "One row per product and route with source, CD, runtime, HTTP, feature and data-freshness evidence in the same window.",
"callback": "Project matrix status to AWOOOI product governance and AwoooP Work Items.",
"acceptance": "All 12 product rows and all public surfaces are either verified current or explicitly degraded with owner and next action.",
"next_action": "Generate a fresh matrix and close route ownership_contract_pending rows."
},
{
"id": "AG99-P0-011",
"order": 11,
"priority": "P0",
"status": "planned",
"title": "Global log registry, anomaly analysis and correlation",
"owner_lane": "aiops_log_intelligence",
"dependencies": ["AG99-P0-002", "AG99-P0-004"],
"scope_refs": ["all:hosts", "all:products", "all:sites", "all:services", "all:tools", "all:packages"],
"problem": "Logs are distributed and noisy; Agent99 lacks a complete freshness-aware source registry and cross-signal causal correlation.",
"trigger": "New log source registration, source freshness breach or incident evidence collection.",
"ai_role": "Log analyst and root-cause correlator",
"action_mode": "observe_then_controlled_apply",
"evidence_sources": ["redacted log metadata", "metrics", "traces", "deploy events", "backup events", "host events"],
"controlled_actions": ["tag and cluster", "identify primary and secondary signals", "create evidence packet", "route to playbook"],
"prohibited_actions": ["ingest secrets", "read raw sessions or SQLite", "auto-execute from untrusted log text"],
"rollback": "Disable faulty adapter or model route while retaining raw-source pointers and normalized receipts.",
"verifier": "Coverage, freshness, redaction, cluster precision and incident linkage readback.",
"callback": "Write cluster, root-cause confidence and evidence refs to incident and KM/RAG.",
"acceptance": "Every registered domain has owner, adapter, freshness and redaction status; primary issue and noise are distinguishable.",
"next_action": "Build metadata-only registry and first adapters for Telegram, reboot, CPU, backup and provider freshness."
},
{
"id": "AG99-P0-012",
"order": 12,
"priority": "P0",
"status": "in_progress",
"title": "Backup observability, integrity, offsite and restore drills",
"owner_lane": "backup_dr",
"dependencies": ["AG99-P0-006", "AG99-P0-011"],
"scope_refs": ["all:backup_domains", "all:hosts", "all:products"],
"problem": "Backup activity and restore readiness are not consistently visible or understandable across host, DB, site, service, package, tool, log and AI artifacts.",
"trigger": "Backup schedule window, freshness breach, integrity failure, offsite failure or restore drill due date.",
"ai_role": "Backup auditor and safe restore planner",
"action_mode": "observe_then_controlled_apply",
"evidence_sources": ["schedule", "last success", "snapshot metadata", "integrity result", "offsite receipt", "restore drill"],
"controlled_actions": ["rerun non-destructive verifier", "repair stale status marker", "queue isolated restore drill"],
"prohibited_actions": ["read backup contents or secrets", "prune", "restore over production", "delete remote copies"],
"rollback": "Stop verifier or isolated drill without changing source repositories.",
"verifier": "Each baseline domain has schedule, freshness, integrity and isolated restore evidence.",
"callback": "Report one lifecycle incident per failed backup domain and update DR scorecard.",
"acceptance": "No unknown backup target; failures name the domain, age, SLA, owner, evidence and safe next action.",
"next_action": "Reconcile failed readbacks and add missing full Gitea, DB, packages, LFS and tool/log restore receipts."
},
{
"id": "AG99-P0-013",
"order": 13,
"priority": "P0",
"status": "in_progress",
"title": "All-host performance AIOps and safe load shedding",
"owner_lane": "performance_aiops",
"dependencies": ["AG99-P0-001", "AG99-P0-005", "AG99-P0-011"],
"scope_refs": ["all:hosts", "all:services", "host:110", "host:188"],
"problem": "110 and 188 have recurring pressure, some readbacks are empty, and one-time load drops are treated as closure without causal proof.",
"trigger": "Sustained CPU, memory, disk, IO, queue or database threshold breach, or a performance readback failure.",
"ai_role": "Performance diagnostician and load-shed controller",
"action_mode": "controlled_apply",
"evidence_sources": ["CPU per core", "memory", "disk", "IO", "process and container attribution", "DB queries", "queue", "backup and crawler schedules"],
"controlled_actions": ["throttle allowlisted batch work", "pause duplicate jobs", "apply existing resource limits", "resume after verifier"],
"prohibited_actions": ["kill unknown process", "restart stateful service generically", "emit empty-value alerts"],
"rollback": "Restore prior quotas or resume paused jobs after health and queue checks.",
"verifier": "Sustained observation window plus service SLO, queue, data freshness and process attribution.",
"callback": "Update performance incident and Telegram only on state change or action result.",
"acceptance": "Pressure is attributed, safely reduced and stays within threshold without service or freshness regression.",
"next_action": "Fix perf readback failures and complete 110/188 process, DB and backup/crawler correlation."
},
{
"id": "AG99-P0-014",
"order": 14,
"priority": "P0",
"status": "in_progress",
"title": "Gitea, Harbor, Registry, Gitea-only CD and complete DR",
"owner_lane": "source_control_registry_cd_dr",
"dependencies": ["AG99-P0-005", "AG99-P0-006", "AG99-P0-012"],
"scope_refs": ["host:110", "all:products", "surface:gitea", "surface:harbor", "surface:registry"],
"problem": "Repository refs are currently present, but complete Gitea state, registry, runner/CD and full restore coverage must be proven continuously.",
"trigger": "SCM, registry or CD health failure; scheduled backup/restore audit; repository visibility complaint; release drift.",
"ai_role": "SCM, registry and release recovery engineer",
"action_mode": "controlled_apply",
"evidence_sources": ["Gitea SSH refs", "main and dev branches", "Gitea full dump", "DB settings issues packages LFS", "Harbor manifest", "runner queue", "deploy marker", "sample restore"],
"controlled_actions": ["repair stateless exporter", "queue bounded runner action", "verify repository and registry", "perform isolated restore dry run"],
"prohibited_actions": ["create replacement repo to hide loss", "change visibility", "force push", "restore production", "use GitHub"],
"rollback": "Restore previous runner/CD config and keep registry/source incident open; never rewrite repository history.",
"verifier": "All expected repos and refs, full dump components, registry artifacts, CD run and runtime markers read back successfully.",
"callback": "Update source-control, release and DR scorecards with one trace.",
"acceptance": "All 12 products have source and release proof and the platform can be reconstructed in isolation from verified backups.",
"next_action": "Add full Gitea state and registry restore receipts to the existing repo-bundle proof."
},
{
"id": "AG99-P1-001",
"order": 15,
"priority": "P1",
"status": "planned",
"title": "Model-backed planner, reviewer and tool loop",
"owner_lane": "agent_ai_runtime",
"dependencies": ["AG99-P0-004", "AG99-P0-005"],
"scope_refs": ["platform:OpenClaw", "platform:Hermes", "platform:NemoTron", "platform:ElephantAlpha", "platform:Ollama"],
"action_mode": "shadow_then_canary",
"verifier": "Replay, shadow agreement, unsafe-action rejection, canary success and cost/latency limits.",
"acceptance": "The planner emits structured allowlisted tool plans with evidence, risk, rollback and verifier, and cannot bypass policy.",
"next_action": "Define planner tool schema and replay corpus from verified incidents."
},
{
"id": "AG99-P1-002",
"order": 16,
"priority": "P1",
"status": "planned",
"title": "PlayBook trust, promotion and recurrence prevention",
"owner_lane": "knowledge_governance",
"dependencies": ["AG99-P0-006", "AG99-P1-001"],
"scope_refs": ["platform:KM", "platform:RAG", "platform:MCP", "platform:PlayBook"],
"action_mode": "controlled_apply",
"verifier": "Trust score is derived from replay, execution, rollback, recurrence and human override receipts.",
"acceptance": "Repeated incidents promote verified automation or create a permanent-fix work item; failed playbooks are demoted automatically.",
"next_action": "Define trust and promotion schema with minimum replay/shadow/canary evidence."
},
{
"id": "AG99-P1-003",
"order": 17,
"priority": "P1",
"status": "planned",
"title": "Human-readable Traditional Chinese Telegram incident cards",
"owner_lane": "telegram_incident_experience",
"dependencies": ["AG99-P0-006"],
"scope_refs": ["surface:Telegram", "all:incidents"],
"action_mode": "controlled_apply",
"verifier": "Message snapshot tests, duplicate suppression, state-change delivery and evidence-link checks.",
"acceptance": "Every card explains impact, AI action, verified result and next step in Traditional Chinese; charts or screenshots are attached when they improve understanding.",
"next_action": "Replace raw technical completion messages with one lifecycle card per incident state change."
},
{
"id": "AG99-P1-004",
"order": 18,
"priority": "P1",
"status": "planned",
"title": "Agent99 desktop, Telegram and API command surfaces",
"owner_lane": "agent99_operator_experience",
"dependencies": ["AG99-P0-003", "AG99-P0-006"],
"scope_refs": ["host:99", "surface:Telegram", "surface:Agent99 desktop"],
"action_mode": "controlled_apply",
"verifier": "The same request has identical policy, trace, execution and result across all command surfaces.",
"acceptance": "Operators can issue natural-language tasks without Codex quota and cannot escape allowlisted tools or audit.",
"next_action": "Unify Submit Request, Telegram and API behind the authenticated task schema."
},
{
"id": "AG99-P1-005",
"order": 19,
"priority": "P1",
"status": "planned",
"title": "AWOOOI, AwoooP and IWOOOS product integration",
"owner_lane": "platform_product_integration",
"dependencies": ["AG99-P0-006", "AG99-P1-002"],
"scope_refs": ["platform:AWOOOI", "platform:AwoooP", "platform:IWOOOS", "platform:AISOC"],
"action_mode": "controlled_apply",
"verifier": "Runs, Work Items, Incidents, Observability and Knowledge show the same asset ids, owners, states and next actions.",
"acceptance": "No remediation exists only in Telegram or Windows files; all lifecycle assets are visible and queryable in the products.",
"next_action": "Add shared incident/run/work-item/asset projections and deep links."
},
{
"id": "AG99-P1-006",
"order": 20,
"priority": "P1",
"status": "planned",
"title": "Tool, package, image and model lifecycle governance",
"owner_lane": "supply_chain_governance",
"dependencies": ["AG99-P0-010", "AG99-P0-014"],
"scope_refs": ["all:tools", "all:packages", "all:container_images", "all:ai_models"],
"action_mode": "observe_then_controlled_apply",
"verifier": "Version, digest, SBOM, CVE, EOL, compatibility, rollback and runtime readback per managed artifact.",
"acceptance": "Drift and vulnerable or unsupported artifacts are visible, prioritized and updated only through canary and rollback-aware plans.",
"next_action": "Build package-domain inventory and freshness policy without automatic major upgrades."
},
{
"id": "AG99-P1-007",
"order": 21,
"priority": "P1",
"status": "planned",
"title": "AISOC security triage and controlled response",
"owner_lane": "iwooos_aisoc",
"dependencies": ["AG99-P0-003", "AG99-P0-011", "AG99-P1-001"],
"scope_refs": ["host:112", "platform:IWOOOS", "platform:AISOC", "all:security_signals"],
"action_mode": "observe_then_controlled_apply",
"verifier": "Evidence integrity, rule/model agreement, containment scope, rollback and service impact checks.",
"acceptance": "Security incidents receive evidence-backed triage and safe containment candidates without autonomous attack scans or credential actions.",
"next_action": "Map IWOOOS/Wazuh signal taxonomy to the shared incident and candidate schema."
},
{
"id": "AG99-P1-008",
"order": 22,
"priority": "P1",
"status": "planned",
"title": "Automation metrics, incident reporting and permanent fixes",
"owner_lane": "agentops_reporting",
"dependencies": ["AG99-P0-006", "AG99-P1-002", "AG99-P1-005"],
"scope_refs": ["all:incidents", "all:work_items", "platform:AgentOps"],
"action_mode": "observe",
"verifier": "Counts reconcile with event and verifier receipts; reports distinguish current, resolved, failed, avoided and recurrent problems.",
"acceptance": "Daily, incident and weekly reports expose MTTA, MTTR, SLO, recurrence, false positives, coverage and human intervention.",
"next_action": "Define counters and report schema from canonical event and verifier data."
},
{
"id": "AG99-P2-001",
"order": 23,
"priority": "P2",
"status": "planned",
"title": "Cold-start, restore and chaos game days",
"owner_lane": "resilience_engineering",
"dependencies": ["AG99-P0-007", "AG99-P0-012", "AG99-P1-008"],
"scope_refs": ["all:hosts", "all:products", "all:backup_domains"],
"action_mode": "isolated_then_controlled_apply",
"verifier": "Preflight, abort criteria, SLO timing, rollback and post-drill data integrity.",
"acceptance": "Recovery claims are proven by scheduled isolated drills and explicitly approved production game days.",
"next_action": "Create isolated drill calendar and evidence templates."
},
{
"id": "AG99-P2-002",
"order": 24,
"priority": "P2",
"status": "planned",
"title": "Capacity, cost and model quality optimization",
"owner_lane": "capacity_finops_agentops",
"dependencies": ["AG99-P0-013", "AG99-P1-001", "AG99-P1-008"],
"scope_refs": ["all:hosts", "all:models", "all:automation_lanes"],
"action_mode": "observe_then_controlled_apply",
"verifier": "Capacity headroom, service SLO, automation success, model quality, latency and approved cost limits.",
"acceptance": "Resource and model routing changes improve measured outcomes without hidden provider or cost changes.",
"next_action": "Establish baselines before any optimization action."
},
{
"id": "AG99-P2-003",
"order": 25,
"priority": "P2",
"status": "planned",
"title": "Controlled GUI, VM Console and browser automation",
"owner_lane": "agent99_desktop_automation",
"dependencies": ["AG99-P0-003", "AG99-P0-008", "AG99-P1-004"],
"scope_refs": ["host:99", "surface:VMware Console", "surface:Windows desktop", "surface:browser"],
"action_mode": "controlled_apply",
"verifier": "App identity, allowed action, before/after screenshot, target-state readback and rollback.",
"acceptance": "GUI automation is used only where no reliable API or CLI exists and remains fully policy-bound and auditable.",
"next_action": "Define an allowlisted desktop tool catalog and screenshot evidence contract."
}
],
"summary": {
"host_count": 7,
"product_count": 12,
"public_surface_count": 23,
"work_item_count": 25,
"p0_count": 14,
"p1_count": 8,
"p2_count": 3,
"in_progress_count": 5,
"in_progress_blocked_count": 3,
"planned_count": 17,
"complete_count": 0,
"current_p0_count": 1
},
"next_execution_order": [
"AG99-P0-001",
"AG99-P0-002",
"AG99-P0-003",
"AG99-P0-004",
"AG99-P0-005",
"AG99-P0-006",
"AG99-P0-007",
"AG99-P0-008",
"AG99-P0-009",
"AG99-P0-010",
"AG99-P0-011",
"AG99-P0-012",
"AG99-P0-013",
"AG99-P0-014"
]
}

View File

@@ -1,5 +1,7 @@
# 2026-06-04 Reboot / Cold-Start / Backup Recovery Workplan
> **2026-07-10 執行入口:** 本文件保留 cold-start、backup 與事故經驗細節目前工作順序、Agent99 控制面、全產品/網站/工具覆蓋與完成契約,以 `docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md` 及其機器可讀總帳為準。重啟恢復仍是 P0但必須先修正 Agent99 假 resolved、routing/dedupe、secure ingress 與 verifier/callback否則不能信任自動結案。
> Owner: SRE / DevOps commander
> Timezone: Asia/Taipei
> Baseline: 2026-06-04 15:00 live read-only checks. Do not reuse the 2026-05-29 baseline without rerunning checks.

View File

@@ -1,5 +1,7 @@
# 統帥插入需求主線優先序台帳
> **2026-07-10 排序更新:** 本文件保留為完整需求與歷史證據來源;目前唯一執行順序、全主機/產品/網站/服務/工具/套件覆蓋與 Agent99 AI 自動化完成契約,改由 `docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md` 與 `docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json` 管理。下方舊狀態不得覆蓋新總帳,也不得再以歷史 `Done` 宣稱目前 runtime 完成。
> 建立時間2026-07-02 13:04 CST
> 範圍:本輪 AWOOOI / Gitea / Product Governance 工作中,使用者在主線推進期間插入的要求。
> 目的:把插入要求轉成可排序、可驗證、可接續的工作項目,避免後續 Codex 把它們當成聊天雜訊或另開支線。

View File

@@ -0,0 +1,160 @@
# Agent99 全域 AI 自動化主計畫
> 建立時間2026-07-10 15:05 CST
> 狀態P0 執行中,尚未達成全域 AI 自動化
> 機器可讀總帳:`docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json`
> 歷史需求來源:`docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md`
## 1. 北極星與完成定義
192.168.0.99 是內網 AI 執行與協調節點。它必須能接收 AWOOOI、AwoooP、IWOOOS、Telegram、主機、VM、服務與網站訊號完成診斷、受控修復、驗證、回報及知識沉澱不能只做排程腳本、固定 regex 分流或告警轉送。
「完整操控」的專業定義是能力範圍完整、政策受控且全程可稽核,不是任意遠端 shell
- 低風險、可回滾、已有 verifier 的動作由 Agent99 自動執行。
- 中風險動作先產生 candidate、check-mode、rollback 與 verifier再由 controlled apply 執行。
- secret 明文、破壞性 DB、主機 reboot、VM power change、firewall/DNS/provider 切換等事故級動作維持 break-glass。
- 任何 `exitCode=0` 只表示命令完成,不能代表問題已解決;只有目標狀態 verifier 通過且事件來源回讀 resolved才能標記 `resolved`
- 99 關機、LAN 中斷或全站失聯時,由獨立外部 watcher 告警並提供 L1 維護頁;不得讓 99 成為自己唯一的監視器。
全域完成必須同時符合:
1. 7 台主機、12 個產品、全部公開路由、基礎服務、工具與套件都有 inventory、owner、freshness 與 verifier。
2. 每個可處置告警都有 incident、evidence、candidate、policy、execution、rollback、verifier、callback 與 learning receipt。
3. 所有產品都有 Gitea source SHA、CD run、deploy marker、runtime image/version、public route 與資料 freshness 的同一時間窗證據。
4. 全主機重啟可被自動偵測,依相依順序恢復,並以首次可用時間與完整綠燈時間證明 10 分鐘 SLO未達標時自動報 blocker 與 ETA。
5. Telegram 只送人看得懂的繁體中文事件卡發生什麼、影響、Agent 做了什麼、結果、下一步;技術證據放按鈕或連結,適合時附圖表或截圖。
6. 相同問題的發生、解決、失敗與避免次數可查;重複問題必須提升 PlayBook trust 或產生永久修復工作項。
## 2. 覆蓋範圍
### 2.1 主機與 VM
| 主機 | 角色與必要能力 | Agent99 責任 |
| --- | --- | --- |
| 192.168.0.99 | Windows、VMware Workstation、Task Scheduler、PowerShell/OpenSSH、Agent99 | Windows/VMware 狀態、VM inventory/autostart、Windows Update 政策、桌面與命令入口、全程 audit |
| 192.168.0.110 | Gitea、Gitea Actions runner、Docker、Harbor/Registry、Prometheus、Alertmanager、Sentry、backup | Source/CD/registry/observability 恢復、CPU/queue 降載、備份與 restore verifier |
| 192.168.0.111 | 必要 VMware guest目前角色與 verified VMX inventory 未完整 | 補 role manifest、VMX、boot/uptime/SSH/exporter、autostart 與服務 verifier |
| 192.168.0.112 | Kali / IWOOOS 安全節點 | 開機、SSH、安全工具健康、受控調查與 AISOC evidence禁止外部攻擊型掃描自動化 |
| 192.168.0.120 | K3s control plane、VIP、AWOOOI workloads | node/pod/rollout/image-pull/cron/Velero 恢復與 verifier |
| 192.168.0.121 | K3s peer、DR drill | peer readiness、workload balance、DR/Velero drill 與 verifier |
| 192.168.0.188 | PostgreSQL、Redis、MOMO、AI/observability/web workloads | data/AI/provider/SignOz/OpenClaw/Ollama/Nginx 健康、效能與備份修復 |
### 2.2 產品與原始碼
目前 Gitea inventory 的 12 個產品全部納入:`awoooi``ewoooc``2026fifa``agent-bounty-protocol``awooogo``stockplatform-v2``vibework``momo-pro-system``tsenyang-website``vtuber``bitan-pharmacy``clawbot-openclaw`。AWOOOI 內的 AwoooP、IWOOOS、AgentOps、AISOC、KM/RAG/MCP 與 OpenClaw 是平台控制面也必須納入版本、runtime、告警與回寫契約。
### 2.3 公開網站與 API
以 cold-start baseline 為最低完整清單AWOOOI web/API/IWOOOS、VibeWork、AwoooGo、2026 FIFA、Agent、MOMO web/health、StockPlatform web/health/API、碧潭大藥局、天揚、VTuber、Gitea、Harbor、Registry、Sentry、SignOz、Langfuse、AIOps。每列都要有 owner product、expected status、L0 maintenance、L1 external fallback、TLS、runtime version 與資料 freshness不能只看 HTTP 200。
### 2.4 服務、工具、套件與資料
- RuntimeWindows services、VMware、systemd、Docker、containerd、K3s、Nginx、PostgreSQL、Redis、MinIO/物件資料、CronJob/Task Scheduler。
- DevSecOpsGitea、Actions runner、Harbor/Registry、ArgoCD/部署標記、Ansible、Velero、restic/rclone、SBOM 與 package inventory。
- ObservabilityPrometheus、Alertmanager、node exporter、cAdvisor、Sentry、SignOz、Langfuse、Wazuh/IWOOOS、Telegram receipt。
- AIAgent99、OpenClaw、Hermes、NemoTron、ElephantAlpha、Ollama、LiteLLM、MCP、KM、RAG、PlayBook、模型與 provider freshness。
- 資料與備份主機設定、DB、repo、registry、網站、套件清單、工具設定、logs、AI artifacts、K3s/Velero、offsite、integrity check 與 restore drill。
## 3. 目標控制面
| 節點 | 權責 |
| --- | --- |
| 外部 watcher / edge | 99 或全內網失聯時仍可產生 down/up 告警、狀態頁與 L1 維護頁;不依賴 99/110/188 |
| AWOOOI | 事件與 incident source of truth、alert normalization、fingerprint、recurrence、產品與 SLO read model |
| AwoooP | Work Item、Run、approval、candidate、rollback、verifier、資產與 owner 總帳 |
| IWOOOS / AISOC | 安全事件 evidence、policy、調查與受控 response高風險行為不得繞過安全邊界 |
| Agent99 | 99/VM/host/service/website 的 evidence collector、planner executor、controlled remediation 與 callback client |
| KM/RAG/MCP/PlayBook | 可重播 evidence、知識、工具介面、已驗證修復與 trust/recurrence 統計 |
| Telegram SRE 戰情室 | 人類命令與事件摘要介面,不是資料庫、任意 shell 或唯一事件來源 |
## 4. 標準 AI 閉環
所有告警與指令必須走同一條鏈:
1. **Sense**:外部 watcher、Alertmanager、Sentry/SignOz、Windows/VMware、host/exporter、logs、backup、public synthetic 收件。
2. **Normalize**:產生 `event_id``incident_id`、fingerprint、source、target、severity、product、host、service、first_seen/last_seen。
3. **Group/Dedupe**:先做 5 分鐘 grouping、single-flight 與同狀態抑噪,再交給 Agent99狀態改變、修復完成與 SLO breach 不抑制。
4. **Collect Evidence**:執行模式專屬 MCP/readback收 metrics、logs、最近變更、source/runtime/version/freshness 與 dependency state。
5. **Plan**:規則 PlayBook 優先AI planner 只產出 allowlisted tool calls、風險、check-mode、rollback、verifier 與 confidence。
6. **Policy**:依動作與 blast radius 判斷 observe、auto-apply、controlled apply 或 break-glass不可由 producer 自行硬設 `controlledApply=true`
7. **Execute**idempotency key、single-flight、timeout、circuit breaker、before evidence、限定 target selector。
8. **Verify**:模式專屬 post-condition、來源事件 resolved、產品/資料 freshness 與回歸 smoke失敗時執行真正 rollback 或明確升級。
9. **Write Back**Agent99 回寫 AWOOOI incident、AwoooP Run/Work Item/Verifier、IWOOOS evidence 與 Telegram 人類事件卡。
10. **Learn**:更新 occurrence/resolved/failed/avoided 計數、MTTA/MTTR、KM/RAG chunk、PlayBook trust、永久修復候選。
合法狀態只有:`received``diagnosing``candidate_ready``executing``verifying``resolved``degraded``blocked``failed``resolved` 必須有 `verifier_passed=true``source_event_resolved=true` 及 evidence reference。
## 5. 更新後的唯一執行順序
### Wave 0先修正 AI 決策真相
| 順序 | ID | 工作 | 狀態 | 驗收 |
| --- | --- | --- | --- | --- |
| 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved` 假綠,對齊 99 runtime script 與 Gitea source SHA | **進行中current P0** | 每種模式以 verifier 決定 outcome舊 runtime/script drift 歸零;不得再把缺證據算 resolved |
| 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping 與重複事件 | 待辦 | `suggestedMode`/kind 優先於 source 字串5 分鐘先 group同 fingerprint 只建立一個 active incident |
| 3 | AG99-P0-003 | 強化 ingressrelay token 必填、Telegram chat+sender allowlist、idempotency/single consumer | 待辦 | 未認證 LAN request 拒絕producer `auto_repair`/policy 被保留;所有 request 可追 trace |
| 4 | AG99-P0-004 | 新增 EvidenceCollect / RepairCandidate / MCP lane | 待辦 | 缺 evidence 的事件會建立可查 candidate不再錯送 Perf/AwoooRepair`INC-*` 可從收件追到 candidate |
### Wave 1完成可驗證修復閉環
| 順序 | ID | 工作 | 狀態 | 驗收 |
| --- | --- | --- | --- | --- |
| 5 | AG99-P0-005 | 模式專屬 verifier、真 rollback、cooldown/circuit-breaker | 待辦 | Recover/Perf/LoadShed/Harbor/AWOOOI/backup/provider 各有 post-condition失敗不寫成功 cooldown |
| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | 待辦 | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂Telegram 可讀回報只從結果物件產生 |
| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | 進行中live readiness 60% | boot_id/uptime/VM power 建立 reboot event依 dependency DAG 恢復;首可用/全綠時間與 blocker/ETA 可查 |
| 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | 進行中 | 111/112/120/121/188 VMX inventory、開機順序、lock repair、guest running、政策 drift 全部 verifier green |
### Wave 2擴至所有產品與運維面
| 順序 | ID | 工作 | 狀態 | 驗收 |
| --- | --- | --- | --- | --- |
| 9 | AG99-P0-009 | 外部 down/up watcher、L0/L1 維護頁與 502 failover | 進行中 | 99/內網全斷仍能告警edge 失聯自動顯示維護頁;恢復後撤回並驗證,不改正式 DNS 作為日常修復 |
| 10 | AG99-P0-010 | 12 產品、全部網站/API 的版本與資料 freshness 矩陣 | 進行中 | 每列具 source SHA、CD、deploy marker、runtime、HTTP、功能 smoke、data timestamp任何缺欄不宣稱最新 |
| 11 | AG99-P0-011 | 全域 log source registry、AI 異常判讀與 correlation | 待辦 | host/product/site/service/package/tool logs 有 freshness、redaction、tag、cluster、primary/secondary/noise 判定 |
| 12 | AG99-P0-012 | 備份 observability、integrity、offsite 與 restore drill | 進行中 | 所有 backup domain 有 schedule/last success/freshness/restore receipt失敗可定位且 TG 只報一次狀態變化 |
| 13 | AG99-P0-013 | 110/188 及全主機效能 AIOps 與安全降載 | 進行中 | CPU/memory/disk/IO/queue/DB attributionallowlisted action 後 verifier無空值 perf alert、無單次下降假結案 |
| 14 | AG99-P0-014 | Gitea/Harbor/Registry/Gitea-only CD 與完整 DR | 進行中 | repo/main/dev、full dump、DB/settings/issues/packages/LFS、registry、sample restore、runner/CD/runtime marker 全部可回讀 |
### Wave 3真正 AI Agent 與產品化
| 順序 | ID | 優先序 | 工作與驗收 |
| --- | --- | --- | --- |
| 15 | AG99-P1-001 | P1 | 接入 OpenClaw/Hermes/NemoTron/ElephantAlpha 的 planner/reviewer先 replay、shadow、canary再提升執行權限 |
| 16 | AG99-P1-002 | P1 | PlayBook trust/promotion成功、失敗、rollback、人工介入與 recurrence 決定自動化等級 |
| 17 | AG99-P1-003 | P1 | Telegram 繁中事件卡一眼摘要、影響、AI 動作、驗證結果、下一步;長證據摺疊,圖表/截圖按事件需要產生 |
| 18 | AG99-P1-004 | P1 | 99 桌面控制中心、Telegram `/agent99`、本機 Submit Request、API command surface 共用同一 policy/audit不依賴 Codex 額度 |
| 19 | AG99-P1-005 | P1 | AwoooP Runs/Work Items/Knowledge 與 AWOOOI Incident/Observability、IWOOOS/AISOC 完整產品化整合 |
| 20 | AG99-P1-006 | P1 | 工具/套件版本、CVE、SBOM、EOL、更新與 rollbackWindows/Linux/container/model/package drift 皆可查 |
| 21 | AG99-P1-007 | P1 | 安全事件 AI triage、evidence preservation、containment candidate外部攻擊、credential 與不可逆隔離維持 break-glass |
| 22 | AG99-P1-008 | P1 | 每日/事故/週報MTTA、MTTR、SLO、recurrence、avoided、false-positive、automation coverage、人工介入與永久修復 |
| 23 | AG99-P2-001 | P2 | 定期 cold-start/restore/chaos drill 與 game day先隔離環境再 production controlled drill |
| 24 | AG99-P2-002 | P2 | capacity、成本、模型品質與自動化成功率趨勢據此調整資源與 agent routing |
| 25 | AG99-P2-003 | P2 | 受控 GUI/VM Console/browser automation只能用 allowlisted app/action、畫面證據與 rollback不以像素點擊取代 API/CLI |
## 6. 目前真相與禁止誤報
2026-07-10 15:04 production readback
- AWOOOI health HTTP 200但整體狀態為 `degraded`,不能稱完整正常。
- Reboot SLO 為 `blocked_reboot_auto_recovery_slo_not_ready`readiness 60%12 個 active blockersprimary 是 `reboot_event_required_host_unreachable`
- Service/data/backup 的部分 readback 已綠,不代表 reboot event、VMware、edge fallback 與 Windows policy 已完成。
- Gitea inventory 有 12 個產品的 main/dev SSH refs這只證明 repo/branch 存在,不證明每個產品 production 都是最新版本。
- Agent99 現況仍是規則式 SRE automation在 planner、tool loop、mode verifier、callback、secure ingress 完成前,不得宣稱完整 AI Agent。
## 7. 工作項強制欄位與治理
每個工作項必須先有:`id`、priority/order/status、scope、owner lane、dependencies、acceptance 與 next action。任何工作項升到 `candidate_ready``in_progress``executing` 前,還必須補齊 problem、trigger、evidence sources、AI role、action mode、controlled actions、prohibited actions、rollback、mode-specific verifier、callback、evidence refs 與 last update缺一不可執行。
執行規則:
1. 同一時間只有一個 `current_p0`;可平行收集證據,但不能改變執行順序。
2. 所有狀態改變寫入 JSON 總帳,再投影到 AWOOOI/AwoooPMarkdown 是人類說明,不是 runtime truth。
3. source/test、Gitea CD、runtime deploy、public/data readback 分層保存,不能用其中一層替代其他層。
4. controlled apply 前必須有 target selector、before evidence、check-mode、rollback、verifier、timeout、single-flight。
5. `resolved` 後仍要跑 freshness/recurrence observation window復發則 reopen 同 fingerprint 並增加 recurrence。
6. GitHub 全面停用;所有 source、CD 與 restore truth 只走 Gitea、本機與 production readback。
## 8. 立即下一步
目前唯一 P0 是 `AG99-P0-001`:先修正 Agent99 假 resolved 與 99 runtime/source drift建立 outcome contract 與 regression tests。完成後依序做 routing/dedupe、secure ingress、RepairCandidate再回到 reboot state machine 的 live rerun如此不是離開重啟主線而是先修正會把重啟與告警錯誤結案的控制核心。