From db9b33cfa70ee54a7a0ac49ce8640298d9b6333a Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 10:53:37 +0800 Subject: [PATCH] fix(agent): route host pressure through bounded diagnosis --- .../services/awooop_ansible_audit_service.py | 21 +++ .../services/awooop_ansible_post_verifier.py | 26 ++++ .../src/services/repair_candidate_service.py | 36 ++++- .../test_host_pressure_ansible_catalog.py | 84 +++++++++++ .../tests/test_repair_candidate_service.py | 28 ++++ .../playbooks/110-host-pressure-readonly.yml | 97 ++++++++++++ .../ops/gitea-queue-hook-backlog-playbook.py | 138 +++++++++++++++++- .../test_gitea_queue_hook_backlog_playbook.py | 138 +++++++++++++++--- 8 files changed, 532 insertions(+), 36 deletions(-) create mode 100644 apps/api/tests/test_host_pressure_ansible_catalog.py create mode 100644 infra/ansible/playbooks/110-host-pressure-readonly.yml diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index 44126f339..7346f7c29 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -56,6 +56,26 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "risk_level": "low", "no_write_only": True, }, + { + "catalog_id": "ansible:110-host-pressure-readonly", + "playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml", + "check_mode_playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml", + "inventory_hosts": ["host_110"], + "domains": ["host_pressure", "gitea", "runner", "postgres", "systemd_control_plane"], + "keywords": [ + "host110sustainedmoderatepressure", + "host-pressure-controller", + "sustained pressure", + "host_resource", + "gitea_service", + "systemd_control_plane", + ], + "supports_check_mode": True, + "auto_apply_enabled": False, + "approval_required": False, + "risk_level": "low", + "no_write_only": True, + }, { "catalog_id": "ansible:110-devops", "playbook_path": "infra/ansible/playbooks/110-devops.yml", @@ -448,6 +468,7 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None "auto_apply_enabled", "approval_required", "risk_level", + "no_write_only", } } if matched: diff --git a/apps/api/src/services/awooop_ansible_post_verifier.py b/apps/api/src/services/awooop_ansible_post_verifier.py index 8887c144b..2d74a0d10 100644 --- a/apps/api/src/services/awooop_ansible_post_verifier.py +++ b/apps/api/src/services/awooop_ansible_post_verifier.py @@ -58,6 +58,32 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = { "systemctl is-active --quiet wazuh-manager.service", ), ), + "ansible:110-host-pressure-readonly": ( + AssetPostcondition( + "host_110_pressure_controller_asset", + "service", + "host_110", + "test -x /home/wooo/scripts/host-sustained-load-controller.py", + ), + AssetPostcondition( + "host_110_gitea_pressure_probe_asset", + "service", + "host_110", + "test -x /home/wooo/scripts/gitea-queue-hook-backlog-playbook.py", + ), + AssetPostcondition( + "host_110_pressure_metrics_freshness", + "metric", + "host_110", + "test -s /home/wooo/node_exporter_textfiles/host_runaway_process.prom", + ), + AssetPostcondition( + "host_110_docker_metrics_freshness", + "metric", + "host_110", + "test -s /home/wooo/node_exporter_textfiles/docker_stats.prom", + ), + ), "ansible:110-devops": ( AssetPostcondition( "host_110_docker_runtime", diff --git a/apps/api/src/services/repair_candidate_service.py b/apps/api/src/services/repair_candidate_service.py index 459c634d0..65221c4e8 100644 --- a/apps/api/src/services/repair_candidate_service.py +++ b/apps/api/src/services/repair_candidate_service.py @@ -1318,13 +1318,35 @@ class RepairCandidateService: ]) elif target_kind == "host_service": service_ref = target or "" - command_template = f"systemctl restart {service_ref}" - rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120" - route = "host_service_route_after_owner_review" - verifier_plan.extend([ - f"systemctl is-active {service_ref}", - f"journalctl -u {service_ref} -n 120 --no-pager", - ]) + if ( + alertname.strip().lower() == "host110sustainedmoderatepressure" + or "host-pressure-controller" in service_ref.lower() + ): + command_template = ( + "/home/wooo/scripts/host-sustained-load-controller.py " + "--host 110 --load5-per-core-threshold 0.75 " + "--hot-container-cpu-threshold 1.0 " + "--container-cpu-threshold 2.0 " + "--process-family-cpu-threshold 50 " + "--metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom " + "--docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom " + "--json" + ) + rollback_template = "no host mutation; retain current services and rerun bounded verifier" + route = "host_pressure_readonly_after_owner_review" + verifier_plan.extend([ + "rerun host-sustained-load-controller.py after 15m cooldown or active CD terminal", + "require alert fingerprint to resolve or decrease without service restart", + "never treat host-pressure-controller as a systemd service", + ]) + else: + command_template = f"systemctl restart {service_ref}" + rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120" + route = "host_service_route_after_owner_review" + verifier_plan.extend([ + f"systemctl is-active {service_ref}", + f"journalctl -u {service_ref} -n 120 --no-pager", + ]) elif target_kind == "database": db_ref = target or "postgres" command_template = ( diff --git a/apps/api/tests/test_host_pressure_ansible_catalog.py b/apps/api/tests/test_host_pressure_ansible_catalog.py new file mode 100644 index 000000000..7ed2a2aa4 --- /dev/null +++ b/apps/api/tests/test_host_pressure_ansible_catalog.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + +from src.services.awooop_ansible_audit_service import ( + _catalog_hints, + get_ansible_catalog_item, +) + +ROOT = Path(__file__).resolve().parents[3] +PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-host-pressure-readonly.yml" + + +def test_host_pressure_alert_prefers_dedicated_no_write_catalog() -> None: + hints = _catalog_hints( + { + "alertname": "Host110SustainedModeratePressure", + "alert_category": "host_resource", + "affected_services": ["host-pressure-controller", "gitea_service"], + "signals": [ + { + "alert_name": "Host110SustainedModeratePressure", + "labels": { + "component": "host-pressure-controller", + "host": "110", + }, + } + ], + }, + None, + ) + + candidates = hints["candidates"] + assert candidates[0]["catalog_id"] == "ansible:110-host-pressure-readonly" + generic = next(row for row in candidates if row["catalog_id"] == "ansible:110-devops") + assert candidates[0]["match_score"] > generic["match_score"] + assert candidates[0]["auto_apply_enabled"] is False + assert candidates[0]["no_write_only"] is True + + +def test_generic_110_fallback_remains_below_exact_host_pressure_route() -> None: + hints = _catalog_hints( + { + "alertname": "Host110SustainedModeratePressure", + "affected_services": ["host-pressure-controller"], + }, + None, + ) + + candidates = hints["candidates"] + dedicated = next( + row + for row in candidates + if row["catalog_id"] == "ansible:110-host-pressure-readonly" + ) + generic = next( + row for row in candidates if row["catalog_id"] == "ansible:110-devops" + ) + assert dedicated["match_score"] > generic["match_score"] + assert generic["matched_keywords"] == ["110"] + + +def test_host_pressure_playbook_is_bounded_and_read_only() -> None: + payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8")) + text = PLAYBOOK.read_text(encoding="utf-8") + catalog = get_ansible_catalog_item("ansible:110-host-pressure-readonly") + + assert payload[0]["hosts"] == "host_110" + assert payload[0]["become"] is False + assert catalog is not None + assert catalog["auto_apply_enabled"] is False + assert catalog["no_write_only"] is True + assert "host-sustained-load-controller.py" in text + assert "gitea-queue-hook-backlog-playbook.py" in text + for forbidden in ( + "systemctl restart", + "docker restart", + "kill -9", + "kubectl delete", + "reboot", + ): + assert forbidden not in text.lower() diff --git a/apps/api/tests/test_repair_candidate_service.py b/apps/api/tests/test_repair_candidate_service.py index 6975b5575..9bfb7c25c 100644 --- a/apps/api/tests/test_repair_candidate_service.py +++ b/apps/api/tests/test_repair_candidate_service.py @@ -531,6 +531,34 @@ def test_promotion_summary_marks_controlled_runtime_when_apply_gate_passes() -> ) +def test_host_pressure_controller_is_never_drafted_as_systemd_restart() -> None: + service = RepairCandidateService() + + template = service._build_playbook_draft_template( + coverage_gap={ + "target_kind": "host_service", + "coverage_key": "host110sustainedmoderatepressure:host-pressure-controller", + "blocking_stage": "service_playbook_coverage", + "required_mcp_evidence_refs": ["metrics_or_logs_window"], + }, + blockers=["playbook_observe_only"], + lane="promote_diagnostic_to_repair_playbook", + alertname="Host110SustainedModeratePressure", + target_resource="host-pressure-controller", + namespace="", + evidence=None, + playbook=None, + ) + + assert template["suggested_route"] == "host_pressure_readonly_after_owner_review" + assert template["repair_command_template"].startswith( + "/home/wooo/scripts/host-sustained-load-controller.py" + ) + assert template["rollback_command_template"].startswith("no host mutation") + assert "systemctl restart" not in template["repair_command_template"] + assert "systemctl is-active" not in " ".join(template["verifier_plan_template"]) + + @pytest.mark.asyncio async def test_postgres_slow_query_gap_prefills_database_owner_review_not_restart() -> None: incident = _incident() diff --git a/infra/ansible/playbooks/110-host-pressure-readonly.yml b/infra/ansible/playbooks/110-host-pressure-readonly.yml new file mode 100644 index 000000000..003f5c7fa --- /dev/null +++ b/infra/ansible/playbooks/110-host-pressure-readonly.yml @@ -0,0 +1,97 @@ +--- +- name: 110 host pressure bounded read-only diagnosis + hosts: host_110 + gather_facts: false + become: false + serial: 1 + any_errors_fatal: true + + vars: + host_pressure_script_dir: /home/wooo/scripts + host_pressure_metrics_file: /home/wooo/node_exporter_textfiles/host_runaway_process.prom + host_pressure_docker_stats_file: /home/wooo/node_exporter_textfiles/docker_stats.prom + + tasks: + - name: Verify the host pressure controller is deployed + ansible.builtin.stat: + path: "{{ host_pressure_script_dir }}/host-sustained-load-controller.py" + register: host_pressure_controller + changed_when: false + + - name: Require the bounded host pressure controller + ansible.builtin.assert: + that: + - host_pressure_controller.stat.exists + - host_pressure_controller.stat.isreg + fail_msg: host_pressure_controller_missing + success_msg: host_pressure_controller_present + changed_when: false + + - name: Classify host pressure without runtime mutation + ansible.builtin.command: + argv: + - /usr/bin/python3 + - "{{ host_pressure_script_dir }}/host-sustained-load-controller.py" + - --host + - "110" + - --load5-per-core-threshold + - "0.75" + - --hot-container-cpu-threshold + - "1.0" + - --container-cpu-threshold + - "2.0" + - --process-family-cpu-threshold + - "50" + - --metrics-file + - "{{ host_pressure_metrics_file }}" + - --docker-stats-file + - "{{ host_pressure_docker_stats_file }}" + - --script-dir + - "{{ host_pressure_script_dir }}" + - --json + register: host_pressure_readback + check_mode: false + changed_when: false + failed_when: host_pressure_readback.rc not in [0, 75] + + - name: Parse the public-safe controller packet + ansible.builtin.set_fact: + host_pressure_packet: "{{ host_pressure_readback.stdout | from_json }}" + changed_when: false + + - name: Run the Gitea rate probe when selected by the controller + ansible.builtin.command: + argv: + - /usr/bin/python3 + - "{{ host_pressure_script_dir }}/gitea-queue-hook-backlog-playbook.py" + - --host + - "110" + - --metrics-file + - "{{ host_pressure_metrics_file }}" + - --docker-stats-file + - "{{ host_pressure_docker_stats_file }}" + - --script-dir + - "{{ host_pressure_script_dir }}" + - --metrics-rate-sample-seconds + - "2" + - --json + register: gitea_pressure_readback + check_mode: false + changed_when: false + failed_when: gitea_pressure_readback.rc not in [0, 75] + when: >- + 'gitea' in (host_pressure_packet.classification | default('') | lower) + or 'gitea' in (host_pressure_packet.next_action | default('') | lower) + + - name: Emit bounded diagnosis receipt + ansible.builtin.debug: + msg: + schema_version: host_110_pressure_ansible_readback_v1 + controller_classification: "{{ host_pressure_packet.classification | default('unknown') }}" + controller_next_action: "{{ host_pressure_packet.next_action | default('unknown') }}" + gitea_probe_executed: "{{ gitea_pressure_readback is not skipped }}" + gitea_probe_returncode: "{{ gitea_pressure_readback.rc | default(none) }}" + host_write_performed: false + systemd_restart_performed: false + docker_restart_performed: false + process_signal_performed: false diff --git a/scripts/ops/gitea-queue-hook-backlog-playbook.py b/scripts/ops/gitea-queue-hook-backlog-playbook.py index 994327aef..3f1b30b15 100644 --- a/scripts/ops/gitea-queue-hook-backlog-playbook.py +++ b/scripts/ops/gitea-queue-hook-backlog-playbook.py @@ -27,7 +27,9 @@ DEFAULT_SCRIPT_DIR = Path("/home/wooo/scripts") DEFAULT_GITEA_METRICS_URL = "http://192.168.0.110:3001/metrics" DEFAULT_GITEA_HEALTH_URL = "http://192.168.0.110:3001/api/healthz" DEFAULT_GITEA_VERSION_URL = "http://192.168.0.110:3001/api/v1/version" -SCHEMA_VERSION = "gitea_queue_hook_backlog_check_mode_v1" +DEFAULT_METRICS_RATE_SAMPLE_SECONDS = 2.0 +SCHEMA_VERSION = "gitea_queue_hook_backlog_check_mode_v2" +ACTIVE_RUN_STATUSES = frozenset({"running", "waiting", "queued", "pending", "in_progress"}) LABEL_RE = re.compile(r"(?P[A-Za-z_][A-Za-z0-9_]*)=\"(?P(?:[^\"\\\\]|\\\\.)*)\"") METRIC_RE = re.compile( @@ -53,13 +55,20 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--gitea-health-url", default=DEFAULT_GITEA_HEALTH_URL) parser.add_argument("--gitea-version-url", default=DEFAULT_GITEA_VERSION_URL) parser.add_argument("--gitea-metrics-file", type=Path) + parser.add_argument("--gitea-metrics-followup-file", type=Path) parser.add_argument("--gitea-health-file", type=Path) parser.add_argument("--gitea-version-file", type=Path) parser.add_argument("--queue-json-file", type=Path) parser.add_argument("--http-timeout-seconds", type=float, default=5.0) + parser.add_argument( + "--metrics-rate-sample-seconds", + type=float, + default=DEFAULT_METRICS_RATE_SAMPLE_SECONDS, + ) parser.add_argument("--hot-container-cpu-threshold", type=float, default=1.0) parser.add_argument("--gitea-family-cpu-threshold", type=float, default=50.0) parser.add_argument("--hooktasks-warning-threshold", type=float, default=1000.0) + parser.add_argument("--hooktasks-growth-warning-threshold", type=float, default=10.0) parser.add_argument("--json", action="store_true") return parser.parse_args() @@ -296,6 +305,42 @@ def selected_gitea_metrics(samples: list[dict[str, Any]]) -> dict[str, Any]: return selected +def metrics_rate_probe( + first_samples: list[dict[str, Any]], + followup_samples: list[dict[str, Any]], + *, + elapsed_seconds: float, +) -> dict[str, Any]: + first_cpu = _sample_value_any(first_samples, "process_cpu_seconds_total") + followup_cpu = _sample_value_any(followup_samples, "process_cpu_seconds_total") + first_hooks = _sample_value_any(first_samples, "gitea_hooktasks") + followup_hooks = _sample_value_any(followup_samples, "gitea_hooktasks") + if ( + elapsed_seconds <= 0 + or first_cpu is None + or followup_cpu is None + or first_hooks is None + or followup_hooks is None + ): + return { + "available": False, + "elapsed_seconds": round(max(0.0, elapsed_seconds), 3), + "process_cpu_cores": None, + "hooktasks_delta": None, + "hooktasks_per_second": None, + } + + cpu_delta = max(0.0, followup_cpu - first_cpu) + hooktasks_delta = max(0.0, followup_hooks - first_hooks) + return { + "available": True, + "elapsed_seconds": round(elapsed_seconds, 3), + "process_cpu_cores": round(cpu_delta / elapsed_seconds, 6), + "hooktasks_delta": round(hooktasks_delta, 3), + "hooktasks_per_second": round(hooktasks_delta / elapsed_seconds, 6), + } + + def queue_readback_summary(path: Path | None) -> dict[str, Any]: if path is None: return {"available": False, "source": "", "latest_visible_cd_run": None} @@ -324,6 +369,13 @@ def queue_readback_summary(path: Path | None) -> dict[str, Any]: } +def _queue_has_active_cd(queue_readback: dict[str, Any]) -> bool: + latest = queue_readback.get("latest_visible_cd_run") + if not isinstance(latest, dict): + return False + return str(latest.get("status") or "").strip().lower() in ACTIVE_RUN_STATUSES + + def build_payload(args: argparse.Namespace) -> dict[str, Any]: host_samples = parse_prometheus_text(read_text(args.metrics_file)) docker_samples = parse_prometheus_text(read_text(args.docker_stats_file)) @@ -337,6 +389,31 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]: path=args.gitea_metrics_file, timeout_seconds=args.http_timeout_seconds, ) + rate_sample_seconds = min(30.0, max(0.0, float(args.metrics_rate_sample_seconds))) + metrics_followup_read: dict[str, Any] = { + "ok": False, + "status_code": None, + "source": "", + "text": "", + "error_type": "rate_probe_not_collected", + } + rate_elapsed_seconds = 0.0 + if metrics_read.get("ok") and args.gitea_metrics_followup_file is not None: + metrics_followup_read = fetch_text_or_file( + url=args.gitea_metrics_url, + path=args.gitea_metrics_followup_file, + timeout_seconds=args.http_timeout_seconds, + ) + rate_elapsed_seconds = rate_sample_seconds + elif metrics_read.get("ok") and args.gitea_metrics_file is None and rate_sample_seconds > 0: + started = time.monotonic() + time.sleep(rate_sample_seconds) + metrics_followup_read = fetch_text_or_file( + url=args.gitea_metrics_url, + path=None, + timeout_seconds=args.http_timeout_seconds, + ) + rate_elapsed_seconds = time.monotonic() - started health_read = fetch_text_or_file( url=args.gitea_health_url, path=args.gitea_health_file, @@ -348,7 +425,15 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]: timeout_seconds=args.http_timeout_seconds, ) gitea_samples = parse_prometheus_text(str(metrics_read.get("text") or "")) + gitea_followup_samples = parse_prometheus_text( + str(metrics_followup_read.get("text") or "") + ) gitea_metrics = selected_gitea_metrics(gitea_samples) + rate_probe = metrics_rate_probe( + gitea_samples, + gitea_followup_samples, + elapsed_seconds=rate_elapsed_seconds, + ) health_json = _json_from_text(str(health_read.get("text") or "")) version_json = _json_from_text(str(version_read.get("text") or "")) families = process_families(host_samples, host=args.host) @@ -386,6 +471,8 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]: gitea_family_cpu = _family_cpu(families, "gitea_service") hooktasks = float(gitea_metrics.get("gitea_hooktasks") or 0.0) health_status = str(health_json.get("status") or "") + queue_readback = queue_readback_summary(args.queue_json_file) + queue_has_active_cd = _queue_has_active_cd(queue_readback) classification = "observing_gitea_pressure_below_threshold" severity = "info" @@ -402,10 +489,14 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]: classification = "blocked_gitea_health_degraded_requires_service_recovery_playbook" severity = "critical" next_action = "run_gitea_service_health_recovery_check_mode_without_restart" + elif queue_has_active_cd: + classification = "observing_gitea_pressure_explained_by_active_cd" + severity = "info" + next_action = "rerun_host_pressure_verifier_after_cd_terminal_or_15m_cooldown" elif active_actions["container_count"] > 0 or active_actions["process_group_count"] > 0: - classification = "blocked_gitea_actions_pressure_requires_runner_queue_packet" - severity = "warning" - next_action = "run_runner_queue_readback_and_keep_110_pressure_gate_fail_closed" + classification = "observing_gitea_actions_pressure_until_terminal_verifier" + severity = "info" + next_action = "rerun_host_pressure_verifier_after_actions_terminal_or_15m_cooldown" elif ( gitea_container_cpu >= args.hot_container_cpu_threshold and hooktasks >= args.hooktasks_warning_threshold @@ -417,9 +508,28 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]: gitea_container_cpu >= args.hot_container_cpu_threshold or gitea_family_cpu >= args.gitea_family_cpu_threshold ): - classification = "blocked_gitea_service_hot_without_actions_backlog" - severity = "warning" - next_action = "run_gitea_metrics_rate_probe_then_select_quota_or_hook_playbook" + if rate_probe.get("available") is not True: + classification = "blocked_gitea_service_hot_requires_metrics_rate_probe" + severity = "warning" + next_action = "run_gitea_metrics_rate_probe_then_select_quota_or_hook_playbook" + elif float(rate_probe.get("hooktasks_delta") or 0.0) >= float( + args.hooktasks_growth_warning_threshold + ): + classification = "blocked_gitea_hook_activity_rate_requires_bounded_queue_probe" + severity = "warning" + next_action = "read_hook_queue_age_from_authorized_export_before_apply" + elif float(rate_probe.get("process_cpu_cores") or 0.0) < float( + args.hot_container_cpu_threshold + ): + classification = "observing_gitea_pressure_cleared_during_rate_probe" + severity = "info" + next_action = "close_if_alert_resolved_on_bounded_post_verifier" + else: + classification = "observing_gitea_healthy_hot_without_backlog_growth" + severity = "info" + next_action = "rerun_host_pressure_verifier_after_15m_cooldown" + + observed_terminal = classification.startswith("observing_") return { "schema_version": SCHEMA_VERSION, @@ -428,6 +538,12 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]: "classification": classification, "severity": severity, "controlled_apply_allowed": False, + "terminal_disposition": ( + "observe_then_bounded_post_verifier" + if observed_terminal + else "repair_candidate_or_evidence_required" + ), + "post_verifier_due_seconds": 900 if observed_terminal else None, "next_action": next_action, "readback": { "docker_stats": docker_status, @@ -449,17 +565,23 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]: "version": str(version_json.get("version") or ""), }, "selected_gitea_metrics": gitea_metrics, + "metrics_rate_probe": { + **rate_probe, + "followup_ok": bool(metrics_followup_read.get("ok")), + "followup_error_type": metrics_followup_read.get("error_type"), + }, "active_actions": active_actions, "gitea_container_cpu_cores": round(gitea_container_cpu, 6), "gitea_process_family_cpu_percent": round(gitea_family_cpu, 3), "top_containers": containers, "top_containers_untrusted": containers_untrusted, "top_process_families": families[:5], - "queue_readback": queue_readback_summary(args.queue_json_file), + "queue_readback": queue_readback, "thresholds": { "hot_container_cpu": args.hot_container_cpu_threshold, "gitea_family_cpu": args.gitea_family_cpu_threshold, "hooktasks_warning": args.hooktasks_warning_threshold, + "hooktasks_growth_warning": args.hooktasks_growth_warning_threshold, }, "metrics_file": str(args.metrics_file), "docker_stats_file": str(args.docker_stats_file), diff --git a/scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py b/scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py index f5f9d070a..5ba821d97 100644 --- a/scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py +++ b/scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py @@ -55,31 +55,53 @@ def _run_playbook( *, host_metrics: list[str], docker_metrics: list[str], + gitea_metrics: list[str] | None = None, + followup_metrics: list[str] | None = None, + queue_payload: dict | None = None, ) -> subprocess.CompletedProcess[str]: gitea_files = _write_common_gitea_files(tmp_path) + if gitea_metrics is not None: + gitea_files["gitea_metrics"].write_text( + "\n".join(gitea_metrics), + encoding="utf-8", + ) host_file = tmp_path / "host.prom" host_file.write_text("\n".join(host_metrics), encoding="utf-8") docker_file = tmp_path / "docker.prom" docker_file.write_text("\n".join(docker_metrics), encoding="utf-8") + followup_file = tmp_path / "gitea-followup.prom" + if followup_metrics is not None: + followup_file.write_text("\n".join(followup_metrics), encoding="utf-8") + queue_file = tmp_path / "queue.json" + if queue_payload is not None: + queue_file.write_text(json.dumps(queue_payload), encoding="utf-8") + + command = [ + sys.executable, + str(PLAYBOOK_PATH), + "--host", + "110", + "--metrics-file", + str(host_file), + "--docker-stats-file", + str(docker_file), + "--gitea-metrics-file", + str(gitea_files["gitea_metrics"]), + "--gitea-health-file", + str(gitea_files["health"]), + "--gitea-version-file", + str(gitea_files["version"]), + "--metrics-rate-sample-seconds", + "5", + "--json", + ] + if followup_metrics is not None: + command.extend(["--gitea-metrics-followup-file", str(followup_file)]) + if queue_payload is not None: + command.extend(["--queue-json-file", str(queue_file)]) return subprocess.run( - [ - sys.executable, - str(PLAYBOOK_PATH), - "--host", - "110", - "--metrics-file", - str(host_file), - "--docker-stats-file", - str(docker_file), - "--gitea-metrics-file", - str(gitea_files["gitea_metrics"]), - "--gitea-health-file", - str(gitea_files["health"]), - "--gitea-version-file", - str(gitea_files["version"]), - "--json", - ], + command, capture_output=True, text=True, ) @@ -112,7 +134,7 @@ def test_gitea_playbook_classifies_hooktask_backlog_without_secret_reads(tmp_pat assert "Authorization" not in result.stdout -def test_gitea_playbook_routes_active_actions_to_runner_queue_packet(tmp_path: Path) -> None: +def test_gitea_playbook_observes_active_actions_until_terminal_verifier(tmp_path: Path) -> None: result = _run_playbook( tmp_path, host_metrics=[ @@ -124,14 +146,88 @@ def test_gitea_playbook_routes_active_actions_to_runner_queue_packet(tmp_path: P docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.4'], ) - assert result.returncode == 75 + assert result.returncode == 0 payload = json.loads(result.stdout) - assert payload["classification"] == "blocked_gitea_actions_pressure_requires_runner_queue_packet" - assert payload["next_action"] == "run_runner_queue_readback_and_keep_110_pressure_gate_fail_closed" + assert payload["classification"] == "observing_gitea_actions_pressure_until_terminal_verifier" + assert payload["next_action"] == ( + "rerun_host_pressure_verifier_after_actions_terminal_or_15m_cooldown" + ) + assert payload["terminal_disposition"] == "observe_then_bounded_post_verifier" assert payload["readback"]["active_actions"]["container_count"] == 1 assert payload["readback"]["active_actions"]["process_group_count"] == 1 +def test_gitea_playbook_observes_healthy_hot_service_without_backlog_growth( + tmp_path: Path, +) -> None: + result = _run_playbook( + tmp_path, + host_metrics=[ + 'awoooi_host_gitea_actions_active_container_count{host="110"} 0', + 'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0', + 'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 65.6', + ], + docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.58'], + gitea_metrics=[ + 'gitea_build_info{version="1.25.5"} 1', + "gitea_hooktasks 383", + "process_cpu_seconds_total 100", + ], + followup_metrics=[ + 'gitea_build_info{version="1.25.5"} 1', + "gitea_hooktasks 383", + "process_cpu_seconds_total 105.1", + ], + ) + + assert result.returncode == 0 + payload = json.loads(result.stdout) + assert payload["classification"] == "observing_gitea_healthy_hot_without_backlog_growth" + assert payload["controlled_apply_allowed"] is False + assert payload["post_verifier_due_seconds"] == 900 + assert payload["readback"]["metrics_rate_probe"] == { + "available": True, + "elapsed_seconds": 5.0, + "followup_error_type": "", + "followup_ok": True, + "hooktasks_delta": 0.0, + "hooktasks_per_second": 0.0, + "process_cpu_cores": 1.02, + } + + +def test_gitea_playbook_observes_publicly_visible_active_cd(tmp_path: Path) -> None: + result = _run_playbook( + tmp_path, + host_metrics=[ + 'awoooi_host_gitea_actions_active_container_count{host="110"} 0', + 'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0', + 'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 65.6', + ], + docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.58'], + queue_payload={ + "top_visible_runs": [ + { + "workflow": "cd.yaml", + "run_id": "4901", + "status": "Running", + "commit_sha": "34e32f6ed6f29952181cd7aa43ca19210d25339b", + } + ] + }, + ) + + assert result.returncode == 0 + payload = json.loads(result.stdout) + assert payload["classification"] == "observing_gitea_pressure_explained_by_active_cd" + assert payload["readback"]["queue_readback"]["latest_visible_cd_run"] == { + "workflow": "cd.yaml", + "run_id": "4901", + "status": "Running", + "commit_sha": "34e32f6ed6f2", + } + + def test_gitea_playbook_rejects_stale_docker_attribution(tmp_path: Path) -> None: result = _run_playbook( tmp_path,