fix(agent): route host pressure through bounded diagnosis
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m29s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m53s

This commit is contained in:
ogt
2026-07-11 10:53:37 +08:00
parent b44190edb8
commit db9b33cfa7
8 changed files with 532 additions and 36 deletions

View File

@@ -55,31 +55,53 @@ def _run_playbook(
*,
host_metrics: list[str],
docker_metrics: list[str],
gitea_metrics: list[str] | None = None,
followup_metrics: list[str] | None = None,
queue_payload: dict | None = None,
) -> subprocess.CompletedProcess[str]:
gitea_files = _write_common_gitea_files(tmp_path)
if gitea_metrics is not None:
gitea_files["gitea_metrics"].write_text(
"\n".join(gitea_metrics),
encoding="utf-8",
)
host_file = tmp_path / "host.prom"
host_file.write_text("\n".join(host_metrics), encoding="utf-8")
docker_file = tmp_path / "docker.prom"
docker_file.write_text("\n".join(docker_metrics), encoding="utf-8")
followup_file = tmp_path / "gitea-followup.prom"
if followup_metrics is not None:
followup_file.write_text("\n".join(followup_metrics), encoding="utf-8")
queue_file = tmp_path / "queue.json"
if queue_payload is not None:
queue_file.write_text(json.dumps(queue_payload), encoding="utf-8")
command = [
sys.executable,
str(PLAYBOOK_PATH),
"--host",
"110",
"--metrics-file",
str(host_file),
"--docker-stats-file",
str(docker_file),
"--gitea-metrics-file",
str(gitea_files["gitea_metrics"]),
"--gitea-health-file",
str(gitea_files["health"]),
"--gitea-version-file",
str(gitea_files["version"]),
"--metrics-rate-sample-seconds",
"5",
"--json",
]
if followup_metrics is not None:
command.extend(["--gitea-metrics-followup-file", str(followup_file)])
if queue_payload is not None:
command.extend(["--queue-json-file", str(queue_file)])
return subprocess.run(
[
sys.executable,
str(PLAYBOOK_PATH),
"--host",
"110",
"--metrics-file",
str(host_file),
"--docker-stats-file",
str(docker_file),
"--gitea-metrics-file",
str(gitea_files["gitea_metrics"]),
"--gitea-health-file",
str(gitea_files["health"]),
"--gitea-version-file",
str(gitea_files["version"]),
"--json",
],
command,
capture_output=True,
text=True,
)
@@ -112,7 +134,7 @@ def test_gitea_playbook_classifies_hooktask_backlog_without_secret_reads(tmp_pat
assert "Authorization" not in result.stdout
def test_gitea_playbook_routes_active_actions_to_runner_queue_packet(tmp_path: Path) -> None:
def test_gitea_playbook_observes_active_actions_until_terminal_verifier(tmp_path: Path) -> None:
result = _run_playbook(
tmp_path,
host_metrics=[
@@ -124,14 +146,88 @@ def test_gitea_playbook_routes_active_actions_to_runner_queue_packet(tmp_path: P
docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.4'],
)
assert result.returncode == 75
assert result.returncode == 0
payload = json.loads(result.stdout)
assert payload["classification"] == "blocked_gitea_actions_pressure_requires_runner_queue_packet"
assert payload["next_action"] == "run_runner_queue_readback_and_keep_110_pressure_gate_fail_closed"
assert payload["classification"] == "observing_gitea_actions_pressure_until_terminal_verifier"
assert payload["next_action"] == (
"rerun_host_pressure_verifier_after_actions_terminal_or_15m_cooldown"
)
assert payload["terminal_disposition"] == "observe_then_bounded_post_verifier"
assert payload["readback"]["active_actions"]["container_count"] == 1
assert payload["readback"]["active_actions"]["process_group_count"] == 1
def test_gitea_playbook_observes_healthy_hot_service_without_backlog_growth(
tmp_path: Path,
) -> None:
result = _run_playbook(
tmp_path,
host_metrics=[
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 65.6',
],
docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.58'],
gitea_metrics=[
'gitea_build_info{version="1.25.5"} 1',
"gitea_hooktasks 383",
"process_cpu_seconds_total 100",
],
followup_metrics=[
'gitea_build_info{version="1.25.5"} 1',
"gitea_hooktasks 383",
"process_cpu_seconds_total 105.1",
],
)
assert result.returncode == 0
payload = json.loads(result.stdout)
assert payload["classification"] == "observing_gitea_healthy_hot_without_backlog_growth"
assert payload["controlled_apply_allowed"] is False
assert payload["post_verifier_due_seconds"] == 900
assert payload["readback"]["metrics_rate_probe"] == {
"available": True,
"elapsed_seconds": 5.0,
"followup_error_type": "",
"followup_ok": True,
"hooktasks_delta": 0.0,
"hooktasks_per_second": 0.0,
"process_cpu_cores": 1.02,
}
def test_gitea_playbook_observes_publicly_visible_active_cd(tmp_path: Path) -> None:
result = _run_playbook(
tmp_path,
host_metrics=[
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 65.6',
],
docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.58'],
queue_payload={
"top_visible_runs": [
{
"workflow": "cd.yaml",
"run_id": "4901",
"status": "Running",
"commit_sha": "34e32f6ed6f29952181cd7aa43ca19210d25339b",
}
]
},
)
assert result.returncode == 0
payload = json.loads(result.stdout)
assert payload["classification"] == "observing_gitea_pressure_explained_by_active_cd"
assert payload["readback"]["queue_readback"]["latest_visible_cd_run"] == {
"workflow": "cd.yaml",
"run_id": "4901",
"status": "Running",
"commit_sha": "34e32f6ed6f2",
}
def test_gitea_playbook_rejects_stale_docker_attribution(tmp_path: Path) -> None:
result = _run_playbook(
tmp_path,