{
  "schema_version": "awoooi_host_cpu_pressure_drain_readback_v1",
  "generated_at": "2026-07-01T23:28:00+08:00",
  "status": "host_188_cpu_recovered_110_cpu_quota_and_alerting_recovered",
  "scope": {
    "hosts": ["188", "110"],
    "incident_family": "post_reboot_host_cpu_pressure",
    "source_branch": "codex/p0-006-stockplatform-postgres-readback-20260630"
  },
  "operation_boundaries": {
    "github_used": false,
    "secret_or_runner_token_read": false,
    "raw_sessions_or_sqlite_read": false,
    "host_reboot_performed": false,
    "docker_daemon_restart_performed": false,
    "nginx_restart_performed": false,
    "k3s_restart_performed": false,
    "database_restart_performed": false,
    "destructive_database_operation_performed": false,
    "workflow_dispatch_performed": false
  },
  "live_actions": [
    {
      "host": "188",
      "target_selector": "pg_dump process group 3170701 / pg_dump pid 3170704",
      "action": "TERM process group",
      "reason": "pg_dump COPY public.asset_coverage_snapshot TO stdout caused k3s-postgres-recovery CPU pressure",
      "rollback": "rerun the controlled backup/export in a low-pressure window",
      "post_apply_readback": "pg_dump process disappeared and long-running COPY query disappeared"
    },
    {
      "host": "188",
      "target_selector": "awoooi-non110-runner.service, awoooi-non110-runner-autostart.path, awoooi-cd-*api-tests, stockplatform-ubuntu-runner",
      "action": "stop runner/test pressure only",
      "reason": "active CD API tests were repeatedly spawning and driving production PostgreSQL CPU",
      "rollback": "restart the non-110 runner only after host pressure guard and source CD gate are green",
      "post_apply_readback": "runner inactive, no awoooi-cd-* containers, k3s-postgres-recovery docker stats 7.81 percent"
    },
    {
      "host": "188",
      "target_selector": "awoooi PostgreSQL role sessions from 192.168.0.120 running old awooop_conversation_event OR lookup",
      "action": "set max_parallel_workers_per_gather=0, statement_timeout=750ms, enable_seqscan=off, and CONNECTION LIMIT 2 for role; cancel active queries and terminate 192.168.0.120 app DB sessions so new sessions inherit the bound",
      "reason": "the live 120 API was still repeatedly issuing the pre-fix OR query and spawning parallel workers after index apply",
      "rollback": "ALTER ROLE awoooi RESET max_parallel_workers_per_gather; ALTER ROLE awoooi RESET statement_timeout; ALTER ROLE awoooi RESET enable_seqscan; ALTER ROLE awoooi CONNECTION LIMIT -1; then let the app reconnect after source split deploys",
      "post_apply_readback": "rolconfig shows max_parallel_workers_per_gather=0, statement_timeout=750ms, enable_seqscan=off, rolconnlimit=2; old OR, automation_operation_log, and local parallel worker counts dropped to 0; k3s-postgres-recovery dropped to 0.98 percent"
    },
    {
      "host": "188",
      "target_selector": "automation_operation_log truth-chain hot path indexes",
      "action": "CREATE INDEX CONCURRENTLY IF NOT EXISTS for incident_id::text, input::text trigram, and output::text trigram",
      "reason": "truth-chain shifted CPU pressure from conversation_event to automation_operation_log after the first circuit breaker",
      "rollback": "DROP INDEX CONCURRENTLY IF EXISTS idx_automation_operation_log_output_text_trgm; DROP INDEX CONCURRENTLY IF EXISTS idx_automation_operation_log_input_text_trgm; DROP INDEX CONCURRENTLY IF EXISTS idx_automation_operation_log_incident_text;",
      "post_apply_readback": "all three indexes read back indisvalid=true and indisready=true"
    },
    {
      "host": "110",
      "target_selector": "gitea Docker container cgroup CPU quota",
      "action": "docker update --cpus=2 gitea",
      "reason": "Gitea repeatedly exceeded the 2-core sustained CPU warning threshold and was allowed to consume up to 3 cores on the 110 control host",
      "rollback": "docker update --cpus=3 gitea",
      "post_apply_readback": "docker inspect reports nanocpus=2000000000; Gitea /api/v1/version returns 1.25.5; docker_container_cpu_limit_cores{container_name=\"gitea\",host=\"110\"}=2"
    }
  ],
  "source_fixes": [
    {
      "path": "scripts/ci/wait-host-web-build-pressure.sh",
      "change": "block CD startup when docker_stats.prom reports k3s-postgres-recovery CPU cores above HOST_WEB_BUILD_PRESSURE_MAX_POSTGRES_CPU_CORES",
      "evidence": "fixture with k3s-postgres-recovery=6.5 cores fails fast before API tests"
    },
    {
      "path": "scripts/ops/systemd-units-textfile-exporter.py",
      "change": "shorten systemctl show timeout, compact error labels, stop probing after timeout budget, and add non-overlap lock",
      "evidence": "pytest covers timeout budget and busy lock"
    },
    {
      "path": "apps/api/src/services/awooop_truth_chain_service.py",
      "change": "split the inbound conversation event lookup into index-friendly branches instead of one large OR predicate",
      "evidence": "focused tests confirm the source_refs/content_preview branches are isolated and the old source_envelope OR pattern does not return"
    },
    {
      "path": "apps/api/migrations/awooop_conversation_event_content_preview_trgm_2026-07-01.sql",
      "change": "add bounded pg_trgm / btree_gin indexes for content_preview fallback lookup",
      "evidence": "live verifier read back both trigram indexes as valid and ready"
    },
    {
      "path": "apps/api/migrations/automation_operation_log_truth_chain_hot_path_indexes_2026-07-01.sql",
      "change": "add automation_operation_log expression/trigram indexes for truth-chain incident/input/output lookups",
      "evidence": "live verifier read back incident_text, input_text_trgm, and output_text_trgm indexes as valid and ready"
    },
    {
      "path": "ops/monitoring/alerts-unified.yml",
      "change": "add Host110SustainedModeratePressure and point sustained-load auto_repair_action to the deployed /home/wooo/scripts controller path",
      "evidence": "Prometheus rule readback shows Host110SustainedModeratePressure loaded; after quota apply it is inactive because load5/core and Gitea CPU are below threshold"
    },
    {
      "path": "scripts/ops/host-sustained-load-controller.py",
      "change": "route Gitea / StockPlatform container CPU quota pressure to source-specific playbook packets even when host load is below the critical load5/core threshold",
      "evidence": "live controller readback after Gitea dropped below threshold returns observing; test fixture with Gitea 2.08 cores returns blocked_gitea_queue_or_hook_backlog_requires_playbook"
    },
    {
      "path": "scripts/ops/backup-alert-label-contract-check.py",
      "change": "make BackupAggregateRunFailed ignore aggregate-only backup_all noise and require component job failed-count evidence",
      "evidence": "BackupAggregateRunFailed is inactive while backup_all aggregate failed_count remains 5 and component_failed=0"
    }
  ],
  "readback": {
    "host_188": {
      "pre_drain_signals": {
        "k3s_postgres_recovery_docker_stats_percent": "658.65",
        "awoooi_cd_api_tests_container": "awoooi-cd-7359-1-api-tests / awoooi-cd-7360-1-api-tests",
        "active_pg_query": "COPY public.asset_coverage_snapshot TO stdout",
        "pg_dump_application_name": "pg_dump"
      },
      "post_drain_signals": {
        "runner_service": "inactive",
        "runner_autostart_path": "inactive",
        "awoooi_cd_containers": "none",
        "pg_dump_process": "none",
        "k3s_postgres_recovery_docker_stats_percent": "7.81",
        "docker_stats_textfile_k3s_postgres_recovery_cores": "0.406900"
      },
      "post_bounded_db_mitigation_signals": {
        "awoooi_role_connection_limit": "2",
        "awoooi_role_config": "max_parallel_workers_per_gather=0, statement_timeout=750ms, enable_seqscan=off",
        "active_old_or_queries": "0",
        "active_automation_operation_log_queries": "0",
        "local_parallel_workers_for_old_or_query": "0",
        "k3s_postgres_recovery_docker_stats_percent": "0.98",
        "host_load_average": "3.48, 4.43, 6.10",
        "remaining_root_cause": "120 live API still runs the pre-fix OR query until the source split is deployed"
      },
      "deployment_readback": {
        "commit": "abc512a7b3911f44c48668d06965423f56471ab1",
        "gitea_cd_run": "4200",
        "gitea_cd_status": "Waiting",
        "source_split_live_on_120": false
      }
    },
      "host_110": {
        "signals": {
          "load5_initial": "39.14",
          "load5_latest": "6.52",
          "load5_per_core_latest": "0.536667",
          "node_procs_running_latest": "not_high_in_latest_readback",
          "gitea_container_cpu_cores_before_quota": "2.1856",
          "gitea_container_cpu_cores_after_quota_textfile": "1.4917",
          "gitea_container_cpu_limit_cores": "2",
          "controller_container_cpu_threshold": "2.0",
          "controller_latest_classification": "observing_load_within_threshold",
          "ssh_control_path": "available",
          "alert_rules": "Host110SustainedModeratePressure loaded; DockerContainerCpuSustainedHigh inactive; BackupAggregateRunFailed inactive",
          "alert_chain": "110 to VIP/120/121 /api/v1/webhooks/alertmanager synthetic no-secret smoke returned HTTP 200",
          "full_stack_cold_start_after_fix": "PASS=96 WARN=0 BLOCKED=0 Result=GREEN"
        },
        "remaining_warnings": [
          "Alertmanager still has non-CPU disk/resource warnings firing: HostDiskUsageHigh, HostOutOfDiskSpace, DockerContainerMissingResourceLimit",
          "Telegram visible delivery depends on AWOOOI API background analysis / TelegramGateway; webhook intake and Telegram health are configured"
        ]
      }
  },
  "verification": {
    "py_compile": "passed: scripts/ops/systemd-units-textfile-exporter.py and apps/api/src/services/awooop_truth_chain_service.py",
    "bash_n": "passed: scripts/ci/wait-host-web-build-pressure.sh scripts/ops/systemd-units-textfile-exporter.py",
    "pytest": [
      "scripts/ops/tests/test_systemd_units_textfile_exporter.py: 2 passed",
      "scripts/ops/tests/test_systemd_units_textfile_exporter.py + ops/runner/test_cd_controlled_runtime_profile.py: 34 passed",
      "truth-chain source split + content-preview migration focused suite: 7 passed",
      "conversation_event + automation_operation_log hot-path migration tests: 6 passed",
      "scripts/ops/tests/test_host_pressure_alert_contract.py + scripts/ops/tests/test_host_runaway_process_exporter.py: 22 passed",
      "backup alert label contract: BACKUP_ALERT_LABEL_CONTRACT_OK",
      "full-stack cold-start check --monitor-read-only: PASS=96 WARN=0 BLOCKED=0"
    ],
    "diff_check": "passed"
  },
  "next_actions": [
    "after deploy convergence, reset the temporary awoooi role max_parallel_workers_per_gather, statement_timeout, enable_seqscan, and connection limit overrides",
    "continue disk capacity cleanup planning for 110/188 HostDiskUsageHigh without destructive prune",
    "verify AWOOOI API TelegramGateway outbound status if owner still does not see webhook notifications",
    "continue 110 Gitea queue / awoooi-host controlled lane recovery without generic runner restore"
  ]
}
