fix(cd): classify nonfatal deploy notification lag

ops/runner/read-public-gitea-actions-queue.py

@@ -424,7 +424,6 @@ def build_readback(
         build_log_classifier["job_succeeded"]
         and build_log_classifier["deploy_marker_pushed"]
         and build_log_classifier["production_deploy_readback_matched"]
-        and build_log_classifier["build_deploy_success_notification_sent"]
     )
     latest_cd_run_ui_blocked = latest_cd_status == "Blocked"
     latest_cd_deploy_readback_succeeded_despite_ui_blocked = bool(
@@ -444,6 +443,9 @@ def build_readback(
     latest_cd_waiting = bool(
         latest_cd_status == "Waiting" and not latest_cd_deploy_evidence_succeeded
     )
+    latest_cd_running = bool(
+        latest_cd_status == "Running" and not latest_cd_deploy_evidence_succeeded
+    )
     host_pressure_waiting_from_stale_jobs = (
         cd_jobs_stale_or_mismatched
         and latest_cd_status in {"Blocked", "Canceled", "Failure"}
@@ -554,12 +556,33 @@ def build_readback(
     current_cd_waiting_behind_harbor_110_repair_running = (
         latest_cd_waiting and harbor_110_repair_running
     )
+    current_cd_running_without_blocker = bool(
+        latest_cd_running
+        and not latest_cd_visible_blocked
+        and not cd_jobs_stale_or_mismatched
+        and not latest_cd_no_matching_runner_label
+        and not workflow_no_matching
+        and not harbor_110_repair_waiting
+        and not harbor_110_repair_running
+        and not effective_cd_failure_classifier
+        and not effective_tests_log_classifier["host_pressure_blocked_or_waiting"]
+        and not build_log_classifier["inflight_classifier"]
+        and not build_log_classifier["harbor_public_route_blocked_or_retrying"]
+    )
     harbor_110_repair_historical_after_latest_cd_success = bool(
         latest_cd_effective_success
         and latest_cd_run_id
         and harbor_110_repair_run_id
         and harbor_110_repair_run_id != latest_cd_run_id
     )
+    effective_harbor_110_repair_waiting = bool(
+        harbor_110_repair_waiting
+        and not harbor_110_repair_historical_after_latest_cd_success
+    )
+    effective_harbor_110_repair_running = bool(
+        harbor_110_repair_running
+        and not harbor_110_repair_historical_after_latest_cd_success
+    )
     effective_remote_ssh_publickey_auth_stalled = bool(
         harbor_110_repair_log_classifier["remote_ssh_publickey_auth_stalled"]
         and not harbor_110_repair_historical_after_latest_cd_success
@@ -629,8 +652,8 @@ def build_readback(
             if not harbor_110_repair_historical_after_latest_cd_success
             else ""
         ),
-        harbor_110_repair_waiting=harbor_110_repair_waiting,
-        harbor_110_repair_running=harbor_110_repair_running,
+        harbor_110_repair_waiting=effective_harbor_110_repair_waiting,
+        harbor_110_repair_running=effective_harbor_110_repair_running,
         harbor_110_repair_failed=effective_harbor_110_repair_failed,
         harbor_110_repair_waiting_after_cd_harbor_blocker=(
             harbor_110_repair_waiting_after_cd_harbor_blocker
@@ -638,6 +661,7 @@ def build_readback(
         ),
         harbor_110_repair_jobs_stale_or_mismatched=(
             effective_harbor_110_repair_jobs_stale_or_mismatched
+            and not current_cd_running_without_blocker
         ),
         harbor_110_repair_jobs_payload_classifier=(
             harbor_110_repair_jobs_payload_classifier
@@ -699,6 +723,7 @@ def build_readback(
         "latest_visible_cd_run_id": latest_cd_run.get("run_id", ""),
         "latest_visible_cd_run_status": latest_cd_status,
         "latest_visible_cd_run_waiting": latest_cd_waiting,
+        "latest_visible_cd_run_running": latest_cd_running,
         "latest_visible_cd_run_blocked": latest_cd_visible_blocked,
         "latest_visible_cd_run_ui_blocked": latest_cd_run_ui_blocked,
         "latest_visible_cd_deploy_evidence_succeeded_from_log": (
@@ -1053,10 +1078,12 @@ def build_readback(
             if harbor_110_repair_no_matching_runner_label
             else "blocked_no_matching_online_runner"
             if workflow_no_matching
+            else "current_main_cd_running"
+            if current_cd_running_without_blocker
             else "harbor_110_repair_waiting_for_runner_or_queue"
-            if harbor_110_repair_waiting
+            if effective_harbor_110_repair_waiting
             else "harbor_110_repair_running"
-            if harbor_110_repair_running
+            if effective_harbor_110_repair_running
             else "blocked_harbor_110_repair_failed"
             if effective_harbor_110_repair_failed
             else "blocked_harbor_110_repair_run"
@@ -1083,6 +1110,10 @@ def build_readback(
             "current_main_cd_run_visible": bool(latest_cd_run),
             "current_main_cd_run_status": latest_cd_status,
             "current_main_cd_run_waiting": latest_cd_waiting,
+            "current_main_cd_run_running": latest_cd_running,
+            "current_main_cd_running_without_blocker": (
+                current_cd_running_without_blocker
+            ),
             "current_cd_waiting_behind_harbor_110_repair_running": (
                 current_cd_waiting_behind_harbor_110_repair_running
             ),

ops/runner/test_read_public_gitea_actions_queue.py

@@ -91,6 +91,40 @@ def _actions_html_single_cd_run() -> str:
 """
 
 
+def _actions_html_running_cd_after_harbor_repair_success() -> str:
+    return """
+<div class="flex-list run-list">
+  <div class="flex-item tw-items-center">
+    <div class="flex-item-leading">
+      <span data-tooltip-content="Running"></span>
+    </div>
+    <div class="flex-item-main">
+      <a class="flex-item-title" title="fix(km): normalize legacy timestamps" href="/wooo/awoooi/actions/runs/4657">
+        fix(km): normalize legacy timestamps
+      </a>
+      <div class="flex-item-body">
+        <span><b>cd.yaml #4657</b>:</span>Commit
+        <a href="/wooo/awoooi/commit/b257f9e98f7f13646987d9ba490a85c63f24c223">b257f9e98f</a>
+      </div>
+    </div>
+  </div>
+  <div class="flex-item tw-items-center">
+    <div class="flex-item-leading">
+      <span data-tooltip-content="Success"></span>
+    </div>
+    <div class="flex-item-main">
+      <a class="flex-item-title" title="fix(reboot): automate windows99 backup search receipt" href="/wooo/awoooi/actions/runs/4652">
+        fix(reboot): automate windows99 backup search receipt
+      </a>
+      <div class="flex-item-body">
+        <span><b>harbor-110-local-repair.yaml #4652</b>:</span>Scheduled
+      </div>
+    </div>
+  </div>
+</div>
+"""
+
+
 def _cd_workflow_actions_html_single_cd_run() -> str:
     return _actions_html_single_cd_run().replace(
         "<span><b>cd.yaml #3863</b>:</span>Commit",
@@ -328,6 +362,16 @@ def _cd_deploy_success_log() -> str:
 """
 
 
+def _cd_deploy_success_with_notification_failure_log() -> str:
+    return """
+2026-07-09T19:33:19.8265526Z [main 57046c9] chore(cd): deploy b257f9e [skip ci]
+2026-07-09T19:33:22.3412803Z    b257f9e..57046c9  main -> main
+2026-07-09T19:34:01.6866643Z Production deploy readback matches this build and GitOps desired image tag (b257f9e98f) on attempt 1/36;matches_main=True
+2026-07-09T19:34:11.9558448Z CI/CD build-deploy success notification failed (non-fatal)
+2026-07-09T19:34:12.5523925Z Job succeeded
+"""
+
+
 def _harbor_retrying_unavailable_log() -> str:
     return """
 2026-06-30T14:49:17.1327272Z harbor_login_attempt=1 registry_v2_status=502
@@ -1401,6 +1445,119 @@ def test_build_readback_downgrades_running_lag_after_deploy_success() -> None:
     )
 
 
+def test_build_readback_treats_notification_failure_as_nonfatal_after_deploy_success() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_running_cd_after_harbor_repair_success(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={"jobs": [], "total_count": 0},
+        harbor_110_repair_jobs_http_status=200,
+        harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(),
+        latest_cd_build_log_http_status=200,
+        latest_cd_build_log_text=_cd_deploy_success_with_notification_failure_log(),
+    )
+
+    assert payload["status"] == "cd_deploy_success_with_run_level_lag"
+    assert (
+        payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"]
+        is True
+    )
+    assert (
+        payload["readback"][
+            "latest_visible_cd_build_deploy_success_notification_from_log"
+        ]
+        is False
+    )
+    assert payload["readback"]["latest_visible_cd_job_succeeded_from_log"] is True
+    assert payload["readback"]["latest_visible_cd_deploy_marker_pushed_from_log"] is True
+    assert (
+        payload["readback"][
+            "latest_visible_cd_production_deploy_readback_matched_from_log"
+        ]
+        is True
+    )
+    assert (
+        payload["readback"]["latest_visible_harbor_110_repair_historical_after_latest_cd_success"]
+        is True
+    )
+    assert (
+        payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"]
+        is True
+    )
+    assert (
+        payload["rollups"]["harbor_110_repair_historical_after_latest_cd_success"]
+        is True
+    )
+
+
+def test_build_readback_keeps_current_cd_running_ahead_of_stale_harbor_jobs_payload() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_running_cd_after_harbor_repair_success(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={"jobs": [], "total_count": 0},
+        harbor_110_repair_jobs_http_status=200,
+        harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(),
+        latest_cd_build_log_http_status=200,
+        latest_cd_build_log_text="",
+    )
+
+    assert payload["status"] == "current_main_cd_running"
+    assert payload["readback"]["latest_visible_cd_run_running"] is True
+    assert (
+        payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"]
+        is True
+    )
+    assert payload["rollups"]["current_main_cd_run_running"] is True
+    assert payload["rollups"]["current_main_cd_running_without_blocker"] is True
+    assert (
+        payload["rollups"]["harbor_110_repair_historical_after_latest_cd_success"]
+        is False
+    )
+    assert (
+        payload["readback"]["safe_next_action_id"]
+        == "continue_public_gitea_queue_readback"
+    )
+
+
+def test_build_readback_keeps_historical_harbor_running_behind_deploy_success_lag() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_cd_waiting_harbor_repair_running(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={"jobs": [], "total_count": 0},
+        harbor_110_repair_jobs_http_status=200,
+        harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(),
+        latest_cd_build_log_http_status=200,
+        latest_cd_build_log_text=_cd_deploy_success_log(),
+    )
+
+    assert payload["status"] == "cd_deploy_success_with_run_level_lag"
+    assert (
+        payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"]
+        is True
+    )
+    assert (
+        payload["readback"]["latest_visible_cd_run_level_lag_after_deploy_success"]
+        is True
+    )
+    assert payload["readback"]["latest_visible_harbor_110_repair_running"] is True
+    assert (
+        payload["readback"]["latest_visible_harbor_110_repair_historical_after_latest_cd_success"]
+        is True
+    )
+    assert (
+        payload["readback"]["safe_next_action_id"]
+        == "continue_public_gitea_queue_readback"
+    )
+
+
 def test_build_readback_classifies_host_pressure_waiting() -> None:
     module = _load_module()
     payload = module.build_readback(