wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(cd): keep older repair failures behind running cd

Browse Source
This commit is contained in:
ogt
2026-07-09 18:23:58 +08:00
parent 5316484e0c
commit c53e9f610f
2 changed files with 180 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
ops/runner/read-public-gitea-actions-queue.py
View File

@@ -448,10 +448,15 @@ def build_readback(
)
host_pressure_waiting_from_stale_jobs = (
cd_jobs_stale_or_mismatched
and not latest_cd_deploy_evidence_succeeded
and latest_cd_status in {"Blocked", "Canceled", "Failure"}
and tests_log_classifier["host_pressure_waiting"]
and not tests_log_classifier["host_pressure_refused"]
)
host_pressure_ignored_after_deploy_success = bool(
latest_cd_deploy_evidence_succeeded
and tests_log_classifier["host_pressure_blocked_or_waiting"]
)
effective_tests_log_classifier = dict(tests_log_classifier)
if host_pressure_waiting_from_stale_jobs:
effective_tests_log_classifier["host_pressure_classifier"] = (
@@ -459,6 +464,18 @@ def build_readback(
)
effective_tests_log_classifier["host_pressure_waiting"] = False
effective_tests_log_classifier["host_pressure_blocked_or_waiting"] = False
elif host_pressure_ignored_after_deploy_success:
effective_tests_log_classifier["host_pressure_classifier"] = (
(
tests_log_classifier["host_pressure_classifier"]
or "host_web_build_pressure"
)
+ "_ignored_after_deploy_success"
)
effective_tests_log_classifier["host_pressure_waiting"] = False
effective_tests_log_classifier["host_pressure_refused"] = False
effective_tests_log_classifier["host_pressure_interrupted"] = False
effective_tests_log_classifier["host_pressure_blocked_or_waiting"] = False
effective_cd_failure_classifier = build_log_classifier["failure_classifier"]
if (
not effective_cd_failure_classifier
@@ -575,41 +592,53 @@ def build_readback(
and harbor_110_repair_run_id
and harbor_110_repair_run_id != latest_cd_run_id
)
harbor_110_repair_historical_before_current_cd_running = bool(
latest_cd_running
and latest_cd_run_id
and harbor_110_repair_run_id
and harbor_110_repair_failed
and _int(harbor_110_repair_run_id) < _int(latest_cd_run_id)
and not build_log_classifier["harbor_public_route_blocked_or_retrying"]
)
harbor_110_repair_ignored_for_current_cd_context = bool(
harbor_110_repair_historical_after_latest_cd_success
or harbor_110_repair_historical_before_current_cd_running
)
effective_harbor_110_repair_waiting = bool(
harbor_110_repair_waiting
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
effective_harbor_110_repair_running = bool(
harbor_110_repair_running
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
effective_remote_ssh_publickey_auth_stalled = bool(
harbor_110_repair_log_classifier["remote_ssh_publickey_auth_stalled"]
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
effective_remote_control_channel_unavailable = bool(
harbor_110_repair_log_classifier["remote_control_channel_unavailable"]
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
effective_remote_local_registry_v2_unavailable = bool(
harbor_110_repair_log_classifier["local_registry_v2_unavailable"]
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
effective_remote_public_registry_v2_unavailable = bool(
harbor_110_repair_log_classifier["public_registry_v2_unavailable"]
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
effective_harbor_110_repair_failed = bool(
harbor_110_repair_failed
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
effective_harbor_110_repair_jobs_stale_or_mismatched = bool(
harbor_110_repair_jobs_stale_or_mismatched
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
effective_harbor_110_repair_visible_failure_jobs_api_stale = bool(
harbor_110_repair_visible_failure_jobs_api_stale
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
harbor_110_repair_blocked_raw = (
harbor_110_repair_status_blocked
@@ -621,14 +650,14 @@ def build_readback(
)
harbor_110_repair_blocked = bool(
harbor_110_repair_blocked_raw
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
)
safe_next_action = _queue_safe_next_action(
latest_cd_waiting=latest_cd_waiting,
latest_cd_status=latest_cd_status,
latest_cd_no_matching_runner_label=latest_cd_no_matching_runner_label,
cd_jobs_stale_or_mismatched=(
cd_jobs_stale_or_mismatched and not latest_cd_success
cd_jobs_stale_or_mismatched and not latest_cd_effective_success
),
cd_jobs_payload_classifier=cd_jobs_payload_classifier,
effective_host_pressure_classifier=effective_tests_log_classifier[
@@ -649,7 +678,7 @@ def build_readback(
],
harbor_110_repair_no_matching_runner_label=(
harbor_110_repair_no_matching_runner_label
if not harbor_110_repair_historical_after_latest_cd_success
if not harbor_110_repair_ignored_for_current_cd_context
else ""
),
harbor_110_repair_waiting=effective_harbor_110_repair_waiting,
@@ -657,7 +686,7 @@ def build_readback(
harbor_110_repair_failed=effective_harbor_110_repair_failed,
harbor_110_repair_waiting_after_cd_harbor_blocker=(
harbor_110_repair_waiting_after_cd_harbor_blocker
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
),
harbor_110_repair_jobs_stale_or_mismatched=(
effective_harbor_110_repair_jobs_stale_or_mismatched
@@ -668,7 +697,7 @@ def build_readback(
),
harbor_110_repair_visible_running_jobs_api_stale=(
harbor_110_repair_visible_running_jobs_api_stale
and not harbor_110_repair_historical_after_latest_cd_success
and not harbor_110_repair_ignored_for_current_cd_context
),
harbor_110_repair_visible_failure_jobs_api_stale=(
effective_harbor_110_repair_visible_failure_jobs_api_stale
@@ -837,6 +866,9 @@ def build_readback(
"latest_visible_harbor_110_repair_historical_after_latest_cd_success": (
harbor_110_repair_historical_after_latest_cd_success
),
"latest_visible_harbor_110_repair_historical_before_current_cd_running": (
harbor_110_repair_historical_before_current_cd_running
),
"latest_visible_harbor_110_repair_status_blocked": (
harbor_110_repair_status_blocked
),
@@ -1015,6 +1047,9 @@ def build_readback(
"latest_visible_cd_host_pressure_log_stale_or_mismatched": (
host_pressure_waiting_from_stale_jobs
),
"latest_visible_cd_host_pressure_ignored_after_deploy_success": (
host_pressure_ignored_after_deploy_success
),
"no_matching_online_runner_visible": bool(no_matching)
or bool(workflow_no_matching_labels),
"top_visible_runs": combined_visible_runs[:10],
@@ -1186,6 +1221,9 @@ def build_readback(
"current_main_cd_host_pressure_log_stale_or_mismatched": (
host_pressure_waiting_from_stale_jobs
),
"current_main_cd_host_pressure_ignored_after_deploy_success": (
host_pressure_ignored_after_deploy_success
),
"no_matching_online_runner_visible": bool(no_matching)
or bool(workflow_no_matching_labels),
"harbor_110_repair_run_visible": bool(latest_harbor_110_repair_run),
@@ -1196,6 +1234,9 @@ def build_readback(
"harbor_110_repair_historical_after_latest_cd_success": (
harbor_110_repair_historical_after_latest_cd_success
),
"harbor_110_repair_historical_before_current_cd_running": (
harbor_110_repair_historical_before_current_cd_running
),
"harbor_110_repair_blocked": harbor_110_repair_blocked,
"harbor_110_repair_blocked_raw": harbor_110_repair_blocked_raw,
"harbor_110_repair_waiting_after_cd_harbor_blocker": (

125
ops/runner/test_read_public_gitea_actions_queue.py
View File

@@ -125,6 +125,40 @@ def _actions_html_running_cd_after_harbor_repair_success() -> str:
"""
def _actions_html_running_cd_after_harbor_repair_failure() -> str:
return """
<div class="flex-list run-list">
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Running"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="fix(cd): classify nonfatal deploy notification lag" href="/wooo/awoooi/actions/runs/4660">
fix(cd): classify nonfatal deploy notification lag
</a>
<div class="flex-item-body">
<span><b>cd.yaml #4660</b>:</span>Commit
<a href="/wooo/awoooi/commit/709724bd89c8d9dbedd8817983fd1d9c29a28480">709724bd89</a>
</div>
</div>
</div>
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Failure"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="fix(km): expose degraded readback reason" href="/wooo/awoooi/actions/runs/4659">
fix(km): expose degraded readback reason
</a>
<div class="flex-item-body">
<span><b>harbor-110-local-repair.yaml #4659</b>:</span>Scheduled
</div>
</div>
</div>
</div>
"""
def _cd_workflow_actions_html_single_cd_run() -> str:
return _actions_html_single_cd_run().replace(
"<span><b>cd.yaml #3863</b>:</span>Commit",
@@ -1445,6 +1479,55 @@ def test_build_readback_downgrades_running_lag_after_deploy_success() -> None:
)
def test_build_readback_ignores_host_pressure_after_deploy_success_lag() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_single_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 3863,
"head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923",
"conclusion": "success",
}
],
"total_count": 1,
},
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_cd_deploy_success_log(),
latest_cd_tests_log_http_status=200,
latest_cd_tests_log_text=_host_pressure_waiting_log(),
)
assert payload["status"] == "cd_deploy_success_with_run_level_lag"
assert payload["readback"]["latest_visible_cd_run_status"] == "Running"
assert (
payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"]
is True
)
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
"host_web_build_pressure_waiting_ignored_after_deploy_success"
)
assert (
payload["readback"]["latest_visible_cd_host_pressure_ignored_after_deploy_success"]
is True
)
assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is False
assert (
payload["readback"]["safe_next_action_id"]
== "continue_public_gitea_queue_readback"
)
assert (
payload["rollups"]["current_main_cd_host_pressure_ignored_after_deploy_success"]
is True
)
assert payload["rollups"]["current_main_cd_host_pressure_waiting"] is False
def test_build_readback_treats_notification_failure_as_nonfatal_after_deploy_success() -> None:
module = _load_module()
payload = module.build_readback(
@@ -1558,6 +1641,48 @@ def test_build_readback_keeps_historical_harbor_running_behind_deploy_success_la
)
def test_build_readback_keeps_older_harbor_failure_behind_current_cd_running() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_running_cd_after_harbor_repair_failure(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 4660,
"name": "tests",
"head_sha": "ac3258524f50735cd350c3de8b4deab4bd56cac2",
"conclusion": "success",
}
],
"total_count": 1,
},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(),
latest_cd_build_log_http_status=404,
latest_cd_build_log_text="",
latest_harbor_110_repair_log_http_status=200,
latest_harbor_110_repair_log_text=(
_harbor_110_repair_publickey_auth_stalled_log()
),
)
assert payload["status"] == "cd_jobs_stale_or_mismatched"
assert (
payload["readback"][
"latest_visible_harbor_110_repair_historical_before_current_cd_running"
]
is True
)
assert payload["readback"]["latest_visible_harbor_110_repair_blocked"] is False
assert (
payload["readback"]["safe_next_action_id"]
== "ignore_stale_cd_jobs_api_payload_and_poll_visible_cd_or_marker"
)
def test_build_readback_classifies_host_pressure_waiting() -> None:
module = _load_module()
payload = module.build_readback(