wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(agent): route host pressure through bounded diagnosis
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 2m29s
Details
CD Pipeline / build-and-deploy (push) Successful in 6m29s
Details
CD Pipeline / post-deploy-checks (push) Successful in 1m53s
Details

Browse Source
This commit is contained in:
ogt
2026-07-11 10:53:37 +08:00
parent b44190edb8
commit db9b33cfa7
8 changed files with 532 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
scripts/ops/gitea-queue-hook-backlog-playbook.py
View File

@@ -27,7 +27,9 @@ DEFAULT_SCRIPT_DIR = Path("/home/wooo/scripts")
DEFAULT_GITEA_METRICS_URL = "http://192.168.0.110:3001/metrics"
DEFAULT_GITEA_HEALTH_URL = "http://192.168.0.110:3001/api/healthz"
DEFAULT_GITEA_VERSION_URL = "http://192.168.0.110:3001/api/v1/version"
SCHEMA_VERSION = "gitea_queue_hook_backlog_check_mode_v1"
DEFAULT_METRICS_RATE_SAMPLE_SECONDS = 2.0
SCHEMA_VERSION = "gitea_queue_hook_backlog_check_mode_v2"
ACTIVE_RUN_STATUSES = frozenset({"running", "waiting", "queued", "pending", "in_progress"})
LABEL_RE = re.compile(r"(?P<key>[A-Za-z_][A-Za-z0-9_]*)=\"(?P<value>(?:[^\"\\\\]|\\\\.)*)\"")
METRIC_RE = re.compile(
@@ -53,13 +55,20 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--gitea-health-url", default=DEFAULT_GITEA_HEALTH_URL)
parser.add_argument("--gitea-version-url", default=DEFAULT_GITEA_VERSION_URL)
parser.add_argument("--gitea-metrics-file", type=Path)
parser.add_argument("--gitea-metrics-followup-file", type=Path)
parser.add_argument("--gitea-health-file", type=Path)
parser.add_argument("--gitea-version-file", type=Path)
parser.add_argument("--queue-json-file", type=Path)
parser.add_argument("--http-timeout-seconds", type=float, default=5.0)
parser.add_argument(
"--metrics-rate-sample-seconds",
type=float,
default=DEFAULT_METRICS_RATE_SAMPLE_SECONDS,
)
parser.add_argument("--hot-container-cpu-threshold", type=float, default=1.0)
parser.add_argument("--gitea-family-cpu-threshold", type=float, default=50.0)
parser.add_argument("--hooktasks-warning-threshold", type=float, default=1000.0)
parser.add_argument("--hooktasks-growth-warning-threshold", type=float, default=10.0)
parser.add_argument("--json", action="store_true")
return parser.parse_args()
@@ -296,6 +305,42 @@ def selected_gitea_metrics(samples: list[dict[str, Any]]) -> dict[str, Any]:
return selected
def metrics_rate_probe(
first_samples: list[dict[str, Any]],
followup_samples: list[dict[str, Any]],
*,
elapsed_seconds: float,
) -> dict[str, Any]:
first_cpu = _sample_value_any(first_samples, "process_cpu_seconds_total")
followup_cpu = _sample_value_any(followup_samples, "process_cpu_seconds_total")
first_hooks = _sample_value_any(first_samples, "gitea_hooktasks")
followup_hooks = _sample_value_any(followup_samples, "gitea_hooktasks")
if (
elapsed_seconds <= 0
or first_cpu is None
or followup_cpu is None
or first_hooks is None
or followup_hooks is None
):
return {
"available": False,
"elapsed_seconds": round(max(0.0, elapsed_seconds), 3),
"process_cpu_cores": None,
"hooktasks_delta": None,
"hooktasks_per_second": None,
}
cpu_delta = max(0.0, followup_cpu - first_cpu)
hooktasks_delta = max(0.0, followup_hooks - first_hooks)
return {
"available": True,
"elapsed_seconds": round(elapsed_seconds, 3),
"process_cpu_cores": round(cpu_delta / elapsed_seconds, 6),
"hooktasks_delta": round(hooktasks_delta, 3),
"hooktasks_per_second": round(hooktasks_delta / elapsed_seconds, 6),
}
def queue_readback_summary(path: Path | None) -> dict[str, Any]:
if path is None:
return {"available": False, "source": "", "latest_visible_cd_run": None}
@@ -324,6 +369,13 @@ def queue_readback_summary(path: Path | None) -> dict[str, Any]:
}
def _queue_has_active_cd(queue_readback: dict[str, Any]) -> bool:
latest = queue_readback.get("latest_visible_cd_run")
if not isinstance(latest, dict):
return False
return str(latest.get("status") or "").strip().lower() in ACTIVE_RUN_STATUSES
def build_payload(args: argparse.Namespace) -> dict[str, Any]:
host_samples = parse_prometheus_text(read_text(args.metrics_file))
docker_samples = parse_prometheus_text(read_text(args.docker_stats_file))
@@ -337,6 +389,31 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
path=args.gitea_metrics_file,
timeout_seconds=args.http_timeout_seconds,
)
rate_sample_seconds = min(30.0, max(0.0, float(args.metrics_rate_sample_seconds)))
metrics_followup_read: dict[str, Any] = {
"ok": False,
"status_code": None,
"source": "",
"text": "",
"error_type": "rate_probe_not_collected",
}
rate_elapsed_seconds = 0.0
if metrics_read.get("ok") and args.gitea_metrics_followup_file is not None:
metrics_followup_read = fetch_text_or_file(
url=args.gitea_metrics_url,
path=args.gitea_metrics_followup_file,
timeout_seconds=args.http_timeout_seconds,
)
rate_elapsed_seconds = rate_sample_seconds
elif metrics_read.get("ok") and args.gitea_metrics_file is None and rate_sample_seconds > 0:
started = time.monotonic()
time.sleep(rate_sample_seconds)
metrics_followup_read = fetch_text_or_file(
url=args.gitea_metrics_url,
path=None,
timeout_seconds=args.http_timeout_seconds,
)
rate_elapsed_seconds = time.monotonic() - started
health_read = fetch_text_or_file(
url=args.gitea_health_url,
path=args.gitea_health_file,
@@ -348,7 +425,15 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
timeout_seconds=args.http_timeout_seconds,
)
gitea_samples = parse_prometheus_text(str(metrics_read.get("text") or ""))
gitea_followup_samples = parse_prometheus_text(
str(metrics_followup_read.get("text") or "")
)
gitea_metrics = selected_gitea_metrics(gitea_samples)
rate_probe = metrics_rate_probe(
gitea_samples,
gitea_followup_samples,
elapsed_seconds=rate_elapsed_seconds,
)
health_json = _json_from_text(str(health_read.get("text") or ""))
version_json = _json_from_text(str(version_read.get("text") or ""))
families = process_families(host_samples, host=args.host)
@@ -386,6 +471,8 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
gitea_family_cpu = _family_cpu(families, "gitea_service")
hooktasks = float(gitea_metrics.get("gitea_hooktasks") or 0.0)
health_status = str(health_json.get("status") or "")
queue_readback = queue_readback_summary(args.queue_json_file)
queue_has_active_cd = _queue_has_active_cd(queue_readback)
classification = "observing_gitea_pressure_below_threshold"
severity = "info"
@@ -402,10 +489,14 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
classification = "blocked_gitea_health_degraded_requires_service_recovery_playbook"
severity = "critical"
next_action = "run_gitea_service_health_recovery_check_mode_without_restart"
elif queue_has_active_cd:
classification = "observing_gitea_pressure_explained_by_active_cd"
severity = "info"
next_action = "rerun_host_pressure_verifier_after_cd_terminal_or_15m_cooldown"
elif active_actions["container_count"] > 0 or active_actions["process_group_count"] > 0:
classification = "blocked_gitea_actions_pressure_requires_runner_queue_packet"
severity = "warning"
next_action = "run_runner_queue_readback_and_keep_110_pressure_gate_fail_closed"
classification = "observing_gitea_actions_pressure_until_terminal_verifier"
severity = "info"
next_action = "rerun_host_pressure_verifier_after_actions_terminal_or_15m_cooldown"
elif (
gitea_container_cpu >= args.hot_container_cpu_threshold
and hooktasks >= args.hooktasks_warning_threshold
@@ -417,9 +508,28 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
gitea_container_cpu >= args.hot_container_cpu_threshold
or gitea_family_cpu >= args.gitea_family_cpu_threshold
):
classification = "blocked_gitea_service_hot_without_actions_backlog"
severity = "warning"
next_action = "run_gitea_metrics_rate_probe_then_select_quota_or_hook_playbook"
if rate_probe.get("available") is not True:
classification = "blocked_gitea_service_hot_requires_metrics_rate_probe"
severity = "warning"
next_action = "run_gitea_metrics_rate_probe_then_select_quota_or_hook_playbook"
elif float(rate_probe.get("hooktasks_delta") or 0.0) >= float(
args.hooktasks_growth_warning_threshold
):
classification = "blocked_gitea_hook_activity_rate_requires_bounded_queue_probe"
severity = "warning"
next_action = "read_hook_queue_age_from_authorized_export_before_apply"
elif float(rate_probe.get("process_cpu_cores") or 0.0) < float(
args.hot_container_cpu_threshold
):
classification = "observing_gitea_pressure_cleared_during_rate_probe"
severity = "info"
next_action = "close_if_alert_resolved_on_bounded_post_verifier"
else:
classification = "observing_gitea_healthy_hot_without_backlog_growth"
severity = "info"
next_action = "rerun_host_pressure_verifier_after_15m_cooldown"
observed_terminal = classification.startswith("observing_")
return {
"schema_version": SCHEMA_VERSION,
@@ -428,6 +538,12 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
"classification": classification,
"severity": severity,
"controlled_apply_allowed": False,
"terminal_disposition": (
"observe_then_bounded_post_verifier"
if observed_terminal
else "repair_candidate_or_evidence_required"
),
"post_verifier_due_seconds": 900 if observed_terminal else None,
"next_action": next_action,
"readback": {
"docker_stats": docker_status,
@@ -449,17 +565,23 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
"version": str(version_json.get("version") or ""),
},
"selected_gitea_metrics": gitea_metrics,
"metrics_rate_probe": {
**rate_probe,
"followup_ok": bool(metrics_followup_read.get("ok")),
"followup_error_type": metrics_followup_read.get("error_type"),
},
"active_actions": active_actions,
"gitea_container_cpu_cores": round(gitea_container_cpu, 6),
"gitea_process_family_cpu_percent": round(gitea_family_cpu, 3),
"top_containers": containers,
"top_containers_untrusted": containers_untrusted,
"top_process_families": families[:5],
"queue_readback": queue_readback_summary(args.queue_json_file),
"queue_readback": queue_readback,
"thresholds": {
"hot_container_cpu": args.hot_container_cpu_threshold,
"gitea_family_cpu": args.gitea_family_cpu_threshold,
"hooktasks_warning": args.hooktasks_warning_threshold,
"hooktasks_growth_warning": args.hooktasks_growth_warning_threshold,
},
"metrics_file": str(args.metrics_file),
"docker_stats_file": str(args.docker_stats_file),

138
scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py
View File

@@ -55,31 +55,53 @@ def _run_playbook(
*,
host_metrics: list[str],
docker_metrics: list[str],
gitea_metrics: list[str] | None = None,
followup_metrics: list[str] | None = None,
queue_payload: dict | None = None,
) -> subprocess.CompletedProcess[str]:
gitea_files = _write_common_gitea_files(tmp_path)
if gitea_metrics is not None:
gitea_files["gitea_metrics"].write_text(
"\n".join(gitea_metrics),
encoding="utf-8",
)
host_file = tmp_path / "host.prom"
host_file.write_text("\n".join(host_metrics), encoding="utf-8")
docker_file = tmp_path / "docker.prom"
docker_file.write_text("\n".join(docker_metrics), encoding="utf-8")
followup_file = tmp_path / "gitea-followup.prom"
if followup_metrics is not None:
followup_file.write_text("\n".join(followup_metrics), encoding="utf-8")
queue_file = tmp_path / "queue.json"
if queue_payload is not None:
queue_file.write_text(json.dumps(queue_payload), encoding="utf-8")
command = [
sys.executable,
str(PLAYBOOK_PATH),
"--host",
"110",
"--metrics-file",
str(host_file),
"--docker-stats-file",
str(docker_file),
"--gitea-metrics-file",
str(gitea_files["gitea_metrics"]),
"--gitea-health-file",
str(gitea_files["health"]),
"--gitea-version-file",
str(gitea_files["version"]),
"--metrics-rate-sample-seconds",
"5",
"--json",
]
if followup_metrics is not None:
command.extend(["--gitea-metrics-followup-file", str(followup_file)])
if queue_payload is not None:
command.extend(["--queue-json-file", str(queue_file)])
return subprocess.run(
[
sys.executable,
str(PLAYBOOK_PATH),
"--host",
"110",
"--metrics-file",
str(host_file),
"--docker-stats-file",
str(docker_file),
"--gitea-metrics-file",
str(gitea_files["gitea_metrics"]),
"--gitea-health-file",
str(gitea_files["health"]),
"--gitea-version-file",
str(gitea_files["version"]),
"--json",
],
command,
capture_output=True,
text=True,
)
@@ -112,7 +134,7 @@ def test_gitea_playbook_classifies_hooktask_backlog_without_secret_reads(tmp_pat
assert "Authorization" not in result.stdout
def test_gitea_playbook_routes_active_actions_to_runner_queue_packet(tmp_path: Path) -> None:
def test_gitea_playbook_observes_active_actions_until_terminal_verifier(tmp_path: Path) -> None:
result = _run_playbook(
tmp_path,
host_metrics=[
@@ -124,14 +146,88 @@ def test_gitea_playbook_routes_active_actions_to_runner_queue_packet(tmp_path: P
docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.4'],
)
assert result.returncode == 75
assert result.returncode == 0
payload = json.loads(result.stdout)
assert payload["classification"] == "blocked_gitea_actions_pressure_requires_runner_queue_packet"
assert payload["next_action"] == "run_runner_queue_readback_and_keep_110_pressure_gate_fail_closed"
assert payload["classification"] == "observing_gitea_actions_pressure_until_terminal_verifier"
assert payload["next_action"] == (
"rerun_host_pressure_verifier_after_actions_terminal_or_15m_cooldown"
)
assert payload["terminal_disposition"] == "observe_then_bounded_post_verifier"
assert payload["readback"]["active_actions"]["container_count"] == 1
assert payload["readback"]["active_actions"]["process_group_count"] == 1
def test_gitea_playbook_observes_healthy_hot_service_without_backlog_growth(
tmp_path: Path,
) -> None:
result = _run_playbook(
tmp_path,
host_metrics=[
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 65.6',
],
docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.58'],
gitea_metrics=[
'gitea_build_info{version="1.25.5"} 1',
"gitea_hooktasks 383",
"process_cpu_seconds_total 100",
],
followup_metrics=[
'gitea_build_info{version="1.25.5"} 1',
"gitea_hooktasks 383",
"process_cpu_seconds_total 105.1",
],
)
assert result.returncode == 0
payload = json.loads(result.stdout)
assert payload["classification"] == "observing_gitea_healthy_hot_without_backlog_growth"
assert payload["controlled_apply_allowed"] is False
assert payload["post_verifier_due_seconds"] == 900
assert payload["readback"]["metrics_rate_probe"] == {
"available": True,
"elapsed_seconds": 5.0,
"followup_error_type": "",
"followup_ok": True,
"hooktasks_delta": 0.0,
"hooktasks_per_second": 0.0,
"process_cpu_cores": 1.02,
}
def test_gitea_playbook_observes_publicly_visible_active_cd(tmp_path: Path) -> None:
result = _run_playbook(
tmp_path,
host_metrics=[
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 65.6',
],
docker_metrics=['docker_container_cpu_cores{host="110",container_name="gitea"} 1.58'],
queue_payload={
"top_visible_runs": [
{
"workflow": "cd.yaml",
"run_id": "4901",
"status": "Running",
"commit_sha": "34e32f6ed6f29952181cd7aa43ca19210d25339b",
}
]
},
)
assert result.returncode == 0
payload = json.loads(result.stdout)
assert payload["classification"] == "observing_gitea_pressure_explained_by_active_cd"
assert payload["readback"]["queue_readback"]["latest_visible_cd_run"] == {
"workflow": "cd.yaml",
"run_id": "4901",
"status": "Running",
"commit_sha": "34e32f6ed6f2",
}
def test_gitea_playbook_rejects_stale_docker_attribution(tmp_path: Path) -> None:
result = _run_playbook(
tmp_path,