fffebf9597
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
182 lines
11 KiB
JSON
182 lines
11 KiB
JSON
{
|
|
"schema_version": "awoooi_host_cpu_pressure_drain_readback_v1",
|
|
"generated_at": "2026-07-01T23:28:00+08:00",
|
|
"status": "host_188_cpu_recovered_110_cpu_quota_and_alerting_recovered",
|
|
"scope": {
|
|
"hosts": ["188", "110"],
|
|
"incident_family": "post_reboot_host_cpu_pressure",
|
|
"source_branch": "codex/p0-006-stockplatform-postgres-readback-20260630"
|
|
},
|
|
"operation_boundaries": {
|
|
"github_used": false,
|
|
"secret_or_runner_token_read": false,
|
|
"raw_sessions_or_sqlite_read": false,
|
|
"host_reboot_performed": false,
|
|
"docker_daemon_restart_performed": false,
|
|
"nginx_restart_performed": false,
|
|
"k3s_restart_performed": false,
|
|
"database_restart_performed": false,
|
|
"destructive_database_operation_performed": false,
|
|
"workflow_dispatch_performed": false
|
|
},
|
|
"live_actions": [
|
|
{
|
|
"host": "188",
|
|
"target_selector": "pg_dump process group 3170701 / pg_dump pid 3170704",
|
|
"action": "TERM process group",
|
|
"reason": "pg_dump COPY public.asset_coverage_snapshot TO stdout caused k3s-postgres-recovery CPU pressure",
|
|
"rollback": "rerun the controlled backup/export in a low-pressure window",
|
|
"post_apply_readback": "pg_dump process disappeared and long-running COPY query disappeared"
|
|
},
|
|
{
|
|
"host": "188",
|
|
"target_selector": "awoooi-non110-runner.service, awoooi-non110-runner-autostart.path, awoooi-cd-*api-tests, stockplatform-ubuntu-runner",
|
|
"action": "stop runner/test pressure only",
|
|
"reason": "active CD API tests were repeatedly spawning and driving production PostgreSQL CPU",
|
|
"rollback": "restart the non-110 runner only after host pressure guard and source CD gate are green",
|
|
"post_apply_readback": "runner inactive, no awoooi-cd-* containers, k3s-postgres-recovery docker stats 7.81 percent"
|
|
},
|
|
{
|
|
"host": "188",
|
|
"target_selector": "awoooi PostgreSQL role sessions from 192.168.0.120 running old awooop_conversation_event OR lookup",
|
|
"action": "set max_parallel_workers_per_gather=0, statement_timeout=750ms, enable_seqscan=off, and CONNECTION LIMIT 2 for role; cancel active queries and terminate 192.168.0.120 app DB sessions so new sessions inherit the bound",
|
|
"reason": "the live 120 API was still repeatedly issuing the pre-fix OR query and spawning parallel workers after index apply",
|
|
"rollback": "ALTER ROLE awoooi RESET max_parallel_workers_per_gather; ALTER ROLE awoooi RESET statement_timeout; ALTER ROLE awoooi RESET enable_seqscan; ALTER ROLE awoooi CONNECTION LIMIT -1; then let the app reconnect after source split deploys",
|
|
"post_apply_readback": "rolconfig shows max_parallel_workers_per_gather=0, statement_timeout=750ms, enable_seqscan=off, rolconnlimit=2; old OR, automation_operation_log, and local parallel worker counts dropped to 0; k3s-postgres-recovery dropped to 0.98 percent"
|
|
},
|
|
{
|
|
"host": "188",
|
|
"target_selector": "automation_operation_log truth-chain hot path indexes",
|
|
"action": "CREATE INDEX CONCURRENTLY IF NOT EXISTS for incident_id::text, input::text trigram, and output::text trigram",
|
|
"reason": "truth-chain shifted CPU pressure from conversation_event to automation_operation_log after the first circuit breaker",
|
|
"rollback": "DROP INDEX CONCURRENTLY IF EXISTS idx_automation_operation_log_output_text_trgm; DROP INDEX CONCURRENTLY IF EXISTS idx_automation_operation_log_input_text_trgm; DROP INDEX CONCURRENTLY IF EXISTS idx_automation_operation_log_incident_text;",
|
|
"post_apply_readback": "all three indexes read back indisvalid=true and indisready=true"
|
|
},
|
|
{
|
|
"host": "110",
|
|
"target_selector": "gitea Docker container cgroup CPU quota",
|
|
"action": "docker update --cpus=2 gitea",
|
|
"reason": "Gitea repeatedly exceeded the 2-core sustained CPU warning threshold and was allowed to consume up to 3 cores on the 110 control host",
|
|
"rollback": "docker update --cpus=3 gitea",
|
|
"post_apply_readback": "docker inspect reports nanocpus=2000000000; Gitea /api/v1/version returns 1.25.5; docker_container_cpu_limit_cores{container_name=\"gitea\",host=\"110\"}=2"
|
|
}
|
|
],
|
|
"source_fixes": [
|
|
{
|
|
"path": "scripts/ci/wait-host-web-build-pressure.sh",
|
|
"change": "block CD startup when docker_stats.prom reports k3s-postgres-recovery CPU cores above HOST_WEB_BUILD_PRESSURE_MAX_POSTGRES_CPU_CORES",
|
|
"evidence": "fixture with k3s-postgres-recovery=6.5 cores fails fast before API tests"
|
|
},
|
|
{
|
|
"path": "scripts/ops/systemd-units-textfile-exporter.py",
|
|
"change": "shorten systemctl show timeout, compact error labels, stop probing after timeout budget, and add non-overlap lock",
|
|
"evidence": "pytest covers timeout budget and busy lock"
|
|
},
|
|
{
|
|
"path": "apps/api/src/services/awooop_truth_chain_service.py",
|
|
"change": "split the inbound conversation event lookup into index-friendly branches instead of one large OR predicate",
|
|
"evidence": "focused tests confirm the source_refs/content_preview branches are isolated and the old source_envelope OR pattern does not return"
|
|
},
|
|
{
|
|
"path": "apps/api/migrations/awooop_conversation_event_content_preview_trgm_2026-07-01.sql",
|
|
"change": "add bounded pg_trgm / btree_gin indexes for content_preview fallback lookup",
|
|
"evidence": "live verifier read back both trigram indexes as valid and ready"
|
|
},
|
|
{
|
|
"path": "apps/api/migrations/automation_operation_log_truth_chain_hot_path_indexes_2026-07-01.sql",
|
|
"change": "add automation_operation_log expression/trigram indexes for truth-chain incident/input/output lookups",
|
|
"evidence": "live verifier read back incident_text, input_text_trgm, and output_text_trgm indexes as valid and ready"
|
|
},
|
|
{
|
|
"path": "ops/monitoring/alerts-unified.yml",
|
|
"change": "add Host110SustainedModeratePressure and point sustained-load auto_repair_action to the deployed /home/wooo/scripts controller path",
|
|
"evidence": "Prometheus rule readback shows Host110SustainedModeratePressure loaded; after quota apply it is inactive because load5/core and Gitea CPU are below threshold"
|
|
},
|
|
{
|
|
"path": "scripts/ops/host-sustained-load-controller.py",
|
|
"change": "route Gitea / StockPlatform container CPU quota pressure to source-specific playbook packets even when host load is below the critical load5/core threshold",
|
|
"evidence": "live controller readback after Gitea dropped below threshold returns observing; test fixture with Gitea 2.08 cores returns blocked_gitea_queue_or_hook_backlog_requires_playbook"
|
|
},
|
|
{
|
|
"path": "scripts/ops/backup-alert-label-contract-check.py",
|
|
"change": "make BackupAggregateRunFailed ignore aggregate-only backup_all noise and require component job failed-count evidence",
|
|
"evidence": "BackupAggregateRunFailed is inactive while backup_all aggregate failed_count remains 5 and component_failed=0"
|
|
}
|
|
],
|
|
"readback": {
|
|
"host_188": {
|
|
"pre_drain_signals": {
|
|
"k3s_postgres_recovery_docker_stats_percent": "658.65",
|
|
"awoooi_cd_api_tests_container": "awoooi-cd-7359-1-api-tests / awoooi-cd-7360-1-api-tests",
|
|
"active_pg_query": "COPY public.asset_coverage_snapshot TO stdout",
|
|
"pg_dump_application_name": "pg_dump"
|
|
},
|
|
"post_drain_signals": {
|
|
"runner_service": "inactive",
|
|
"runner_autostart_path": "inactive",
|
|
"awoooi_cd_containers": "none",
|
|
"pg_dump_process": "none",
|
|
"k3s_postgres_recovery_docker_stats_percent": "7.81",
|
|
"docker_stats_textfile_k3s_postgres_recovery_cores": "0.406900"
|
|
},
|
|
"post_bounded_db_mitigation_signals": {
|
|
"awoooi_role_connection_limit": "2",
|
|
"awoooi_role_config": "max_parallel_workers_per_gather=0, statement_timeout=750ms, enable_seqscan=off",
|
|
"active_old_or_queries": "0",
|
|
"active_automation_operation_log_queries": "0",
|
|
"local_parallel_workers_for_old_or_query": "0",
|
|
"k3s_postgres_recovery_docker_stats_percent": "0.98",
|
|
"host_load_average": "3.48, 4.43, 6.10",
|
|
"remaining_root_cause": "120 live API still runs the pre-fix OR query until the source split is deployed"
|
|
},
|
|
"deployment_readback": {
|
|
"commit": "abc512a7b3911f44c48668d06965423f56471ab1",
|
|
"gitea_cd_run": "4200",
|
|
"gitea_cd_status": "Waiting",
|
|
"source_split_live_on_120": false
|
|
}
|
|
},
|
|
"host_110": {
|
|
"signals": {
|
|
"load5_initial": "39.14",
|
|
"load5_latest": "6.52",
|
|
"load5_per_core_latest": "0.536667",
|
|
"node_procs_running_latest": "not_high_in_latest_readback",
|
|
"gitea_container_cpu_cores_before_quota": "2.1856",
|
|
"gitea_container_cpu_cores_after_quota_textfile": "1.4917",
|
|
"gitea_container_cpu_limit_cores": "2",
|
|
"controller_container_cpu_threshold": "2.0",
|
|
"controller_latest_classification": "observing_load_within_threshold",
|
|
"ssh_control_path": "available",
|
|
"alert_rules": "Host110SustainedModeratePressure loaded; DockerContainerCpuSustainedHigh inactive; BackupAggregateRunFailed inactive",
|
|
"alert_chain": "110 to VIP/120/121 /api/v1/webhooks/alertmanager synthetic no-secret smoke returned HTTP 200",
|
|
"full_stack_cold_start_after_fix": "PASS=96 WARN=0 BLOCKED=0 Result=GREEN"
|
|
},
|
|
"remaining_warnings": [
|
|
"Alertmanager still has non-CPU disk/resource warnings firing: HostDiskUsageHigh, HostOutOfDiskSpace, DockerContainerMissingResourceLimit",
|
|
"Telegram visible delivery depends on AWOOOI API background analysis / TelegramGateway; webhook intake and Telegram health are configured"
|
|
]
|
|
}
|
|
},
|
|
"verification": {
|
|
"py_compile": "passed: scripts/ops/systemd-units-textfile-exporter.py and apps/api/src/services/awooop_truth_chain_service.py",
|
|
"bash_n": "passed: scripts/ci/wait-host-web-build-pressure.sh scripts/ops/systemd-units-textfile-exporter.py",
|
|
"pytest": [
|
|
"scripts/ops/tests/test_systemd_units_textfile_exporter.py: 2 passed",
|
|
"scripts/ops/tests/test_systemd_units_textfile_exporter.py + ops/runner/test_cd_controlled_runtime_profile.py: 34 passed",
|
|
"truth-chain source split + content-preview migration focused suite: 7 passed",
|
|
"conversation_event + automation_operation_log hot-path migration tests: 6 passed",
|
|
"scripts/ops/tests/test_host_pressure_alert_contract.py + scripts/ops/tests/test_host_runaway_process_exporter.py: 22 passed",
|
|
"backup alert label contract: BACKUP_ALERT_LABEL_CONTRACT_OK",
|
|
"full-stack cold-start check --monitor-read-only: PASS=96 WARN=0 BLOCKED=0"
|
|
],
|
|
"diff_check": "passed"
|
|
},
|
|
"next_actions": [
|
|
"after deploy convergence, reset the temporary awoooi role max_parallel_workers_per_gather, statement_timeout, enable_seqscan, and connection limit overrides",
|
|
"continue disk capacity cleanup planning for 110/188 HostDiskUsageHigh without destructive prune",
|
|
"verify AWOOOI API TelegramGateway outbound status if owner still does not see webhook notifications",
|
|
"continue 110 Gitea queue / awoooi-host controlled lane recovery without generic runner restore"
|
|
]
|
|
}
|