fix(agent): route host pressure through bounded diagnosis
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m29s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m53s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m29s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m53s
This commit is contained in:
@@ -55,31 +55,53 @@ def _run_playbook(
|
||||
*,
|
||||
host_metrics: list[str],
|
||||
docker_metrics: list[str],
|
||||
gitea_metrics: list[str] | None = None,
|
||||
followup_metrics: list[str] | None = None,
|
||||
queue_payload: dict | None = None,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
gitea_files = _write_common_gitea_files(tmp_path)
|
||||
if gitea_metrics is not None:
|
||||
gitea_files["gitea_metrics"].write_text(
|
||||
"\n".join(gitea_metrics),
|
||||
encoding="utf-8",
|
||||
)
|
||||
host_file = tmp_path / "host.prom"
|
||||
host_file.write_text("\n".join(host_metrics), encoding="utf-8")
|
||||
docker_file = tmp_path / "docker.prom"
|
||||
docker_file.write_text("\n".join(docker_metrics), encoding="utf-8")
|
||||
followup_file = tmp_path / "gitea-followup.prom"
|
||||
if followup_metrics is not None:
|
||||
followup_file.write_text("\n".join(followup_metrics), encoding="utf-8")
|
||||
queue_file = tmp_path / "queue.json"
|
||||
if queue_payload is not None:
|
||||
queue_file.write_text(json.dumps(queue_payload), encoding="utf-8")
|
||||
|
||||
command = [
|
||||
sys.executable,
|
||||
str(PLAYBOOK_PATH),
|
||||
"--host",
|
||||
"110",
|
||||
"--metrics-file",
|
||||
str(host_file),
|
||||
"--docker-stats-file",
|
||||
str(docker_file),
|
||||
"--gitea-metrics-file",
|
||||
str(gitea_files["gitea_metrics"]),
|
||||
"--gitea-health-file",
|
||||
str(gitea_files["health"]),
|
||||
"--gitea-version-file",
|
||||
str(gitea_files["version"]),
|
||||
"--metrics-rate-sample-seconds",
|
||||
"5",
|
||||
"--json",
|
||||
]
|
||||
if followup_metrics is not None:
|
||||
command.extend(["--gitea-metrics-followup-file", str(followup_file)])
|
||||
if queue_payload is not None:
|
||||
command.extend(["--queue-json-file", str(queue_file)])
|
||||
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(PLAYBOOK_PATH),
|
||||
"--host",
|
||||
"110",
|
||||
"--metrics-file",
|
||||
str(host_file),
|
||||
"--docker-stats-file",
|
||||
str(docker_file),
|
||||
"--gitea-metrics-file",
|
||||
str(gitea_files["gitea_metrics"]),
|
||||
"--gitea-health-file",
|
||||
str(gitea_files["health"]),
|
||||
"--gitea-version-file",
|
||||
str(gitea_files["version"]),
|
||||
"--json",
|
||||
],
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
@@ -112,7 +134,7 @@ def test_gitea_playbook_classifies_hooktask_backlog_without_secret_reads(tmp_pat
|
||||
assert "Authorization" not in result.stdout
|
||||
|
||||
|
||||
def test_gitea_playbook_routes_active_actions_to_runner_queue_packet(tmp_path: Path) -> None:
|
||||
def test_gitea_playbook_observes_active_actions_until_terminal_verifier(tmp_path: Path) -> None:
|
||||
result = _run_playbook(
|
||||
tmp_path,
|
||||
host_metrics=[
|
||||
@@ -124,14 +146,88 @@ def test_gitea_playbook_routes_active_actions_to_runner_queue_packet(tmp_path: P
|
||||
docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.4'],
|
||||
)
|
||||
|
||||
assert result.returncode == 75
|
||||
assert result.returncode == 0
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["classification"] == "blocked_gitea_actions_pressure_requires_runner_queue_packet"
|
||||
assert payload["next_action"] == "run_runner_queue_readback_and_keep_110_pressure_gate_fail_closed"
|
||||
assert payload["classification"] == "observing_gitea_actions_pressure_until_terminal_verifier"
|
||||
assert payload["next_action"] == (
|
||||
"rerun_host_pressure_verifier_after_actions_terminal_or_15m_cooldown"
|
||||
)
|
||||
assert payload["terminal_disposition"] == "observe_then_bounded_post_verifier"
|
||||
assert payload["readback"]["active_actions"]["container_count"] == 1
|
||||
assert payload["readback"]["active_actions"]["process_group_count"] == 1
|
||||
|
||||
|
||||
def test_gitea_playbook_observes_healthy_hot_service_without_backlog_growth(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result = _run_playbook(
|
||||
tmp_path,
|
||||
host_metrics=[
|
||||
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
|
||||
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
|
||||
'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 65.6',
|
||||
],
|
||||
docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.58'],
|
||||
gitea_metrics=[
|
||||
'gitea_build_info{version="1.25.5"} 1',
|
||||
"gitea_hooktasks 383",
|
||||
"process_cpu_seconds_total 100",
|
||||
],
|
||||
followup_metrics=[
|
||||
'gitea_build_info{version="1.25.5"} 1',
|
||||
"gitea_hooktasks 383",
|
||||
"process_cpu_seconds_total 105.1",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["classification"] == "observing_gitea_healthy_hot_without_backlog_growth"
|
||||
assert payload["controlled_apply_allowed"] is False
|
||||
assert payload["post_verifier_due_seconds"] == 900
|
||||
assert payload["readback"]["metrics_rate_probe"] == {
|
||||
"available": True,
|
||||
"elapsed_seconds": 5.0,
|
||||
"followup_error_type": "",
|
||||
"followup_ok": True,
|
||||
"hooktasks_delta": 0.0,
|
||||
"hooktasks_per_second": 0.0,
|
||||
"process_cpu_cores": 1.02,
|
||||
}
|
||||
|
||||
|
||||
def test_gitea_playbook_observes_publicly_visible_active_cd(tmp_path: Path) -> None:
|
||||
result = _run_playbook(
|
||||
tmp_path,
|
||||
host_metrics=[
|
||||
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
|
||||
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
|
||||
'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 65.6',
|
||||
],
|
||||
docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.58'],
|
||||
queue_payload={
|
||||
"top_visible_runs": [
|
||||
{
|
||||
"workflow": "cd.yaml",
|
||||
"run_id": "4901",
|
||||
"status": "Running",
|
||||
"commit_sha": "34e32f6ed6f29952181cd7aa43ca19210d25339b",
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["classification"] == "observing_gitea_pressure_explained_by_active_cd"
|
||||
assert payload["readback"]["queue_readback"]["latest_visible_cd_run"] == {
|
||||
"workflow": "cd.yaml",
|
||||
"run_id": "4901",
|
||||
"status": "Running",
|
||||
"commit_sha": "34e32f6ed6f2",
|
||||
}
|
||||
|
||||
|
||||
def test_gitea_playbook_rejects_stale_docker_attribution(tmp_path: Path) -> None:
|
||||
result = _run_playbook(
|
||||
tmp_path,
|
||||
|
||||
Reference in New Issue
Block a user