wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(sre): add 24h alert learning corpus

Browse Source
This commit is contained in:
Your Name
2026-07-22 21:54:31 +08:00
parent df50718ac5
commit 3243bc33e0
5 changed files with 467 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
apps/api/tests/test_automation_slo_scorecard.py
View File

@@ -1,3 +1,4 @@
import json
from datetime import UTC, datetime, timedelta
from unittest.mock import AsyncMock
@@ -20,8 +21,10 @@ def _sample(
action_seconds: int,
resolve_seconds: int,
positive: bool,
sample_id: str | None = None,
) -> dict:
return {
"sample_id": sample_id or f"{lane}:{int(start.timestamp())}",
"lane": lane,
"received_at": start,
"first_action_at": start + timedelta(seconds=action_seconds),
@@ -31,7 +34,10 @@ def _sample(
"recurrence_observed": True,
"recurrence": positive,
"human_intervention": positive,
"repair_observed": True,
"repair_accepted": True,
"execution_observed": True,
"shadow_or_no_write_execution_observed": False,
"verifier_observed": True,
"verifier_passed": not positive,
"rollback_performed": positive,
@@ -53,6 +59,7 @@ def test_scorecard_computes_all_required_metrics_and_provider_cost() -> None:
action_seconds=10,
resolve_seconds=100,
positive=False,
sample_id="INC-REAL-1",
),
_sample(
start + timedelta(hours=2),
@@ -60,6 +67,7 @@ def test_scorecard_computes_all_required_metrics_and_provider_cost() -> None:
action_seconds=20,
resolve_seconds=200,
positive=True,
sample_id="INC-REAL-2",
),
],
run_summary={
@@ -115,6 +123,11 @@ def test_scorecard_computes_all_required_metrics_and_provider_cost() -> None:
assert scorecard["runs"]["completed_runs"] == 4
assert scorecard["runs"]["controlled_completed_runs"] == 1
assert scorecard["runs"]["controlled_failed_runs"] == 1
assert scorecard["denominators"]["repair_observed_count"] == 2
assert scorecard["denominators"]["repair_accepted_count"] == 2
assert scorecard["denominators"]["execution_observed_count"] == 2
assert scorecard["denominators"]["verifier_observed_count"] == 2
assert scorecard["denominators"]["shadow_or_no_write_execution_excluded_count"] == 0
assert scorecard["provider_usage"] == {
"rows": [
{
@@ -139,6 +152,40 @@ def test_scorecard_computes_all_required_metrics_and_provider_cost() -> None:
"completion_tokens": 350,
"cost_usd": "0.0123",
}
corpus = scorecard["learning_corpus"]
assert corpus["schema_version"] == "sre_alert_learning_corpus_v1"
assert corpus["window"] == {
"start": start.astimezone(UTC).isoformat(),
"end": end.astimezone(UTC).isoformat(),
}
assert corpus["runtime_closed"] is False
assert corpus["read_only"] is True
assert corpus["raw_context_included"] is False
assert corpus["record_count"] == 2
assert corpus["learning_eligible_record_count"] == 1
assert corpus["status"] == "partial_missing_receipt_coverage"
assert corpus["denominators"] == {
"alert_count": 2,
"repair_observed_count": 2,
"repair_accepted_count": 2,
"production_execution_observed_count": 2,
"verifier_observed_execution_count": 2,
"recurrence_classified_count": 2,
"controlled_non_shadow_run_count": 2,
"provider_call_count": 4,
}
assert corpus["provider_usage"] == scorecard["provider_usage"]
assert corpus["provider_cost_usd"] == "0.0123"
assert corpus["excluded_from_execution"] == {
"shadow_run_count": 2,
"non_shadow_no_step_run_count": 1,
"alert_count_with_shadow_or_no_write_execution_evidence": 0,
}
assert all(row["record_id"].startswith("sha256:") for row in corpus["records"])
assert len(corpus["corpus_sha256"]) == 64
serialized_corpus = json.dumps(corpus)
assert "INC-REAL-1" not in serialized_corpus
assert "INC-REAL-2" not in serialized_corpus
assert scorecard["evidence"] == {
"source": "postgresql_read_only_aggregate",
"same_run_receipt_verification": "pending",
@@ -171,6 +218,90 @@ def test_scorecard_does_not_false_green_missing_runtime_samples() -> None:
"verifier_pass_rate",
}
assert scorecard["provider_usage"]["cost_usd"] == "0.0000"
assert scorecard["learning_corpus"]["status"] == ("no_runtime_sample_fail_closed")
assert scorecard["learning_corpus"]["record_count"] == 0
def test_learning_corpus_excludes_shadow_and_no_write_from_execution() -> None:
end = datetime(2026, 7, 19, 4, 0, tzinfo=UTC)
real = _sample(
end - timedelta(hours=2),
lane="kubernetes_workload",
action_seconds=10,
resolve_seconds=100,
positive=False,
sample_id="INC-PRODUCTION",
)
excluded = _sample(
end - timedelta(hours=1),
lane="host_systemd",
action_seconds=20,
resolve_seconds=200,
positive=False,
sample_id="INC-SHADOW",
)
excluded["execution_observed"] = False
excluded["shadow_or_no_write_execution_observed"] = True
scorecard = build_automation_slo_scorecard(
project_id="awoooi",
incident_samples=[real, excluded],
run_summary={
"total_runs": 3,
"non_shadow_runs": 2,
"controlled_runs_with_steps": 1,
"completed_runs": 3,
"failed_runs": 0,
"controlled_completed_runs": 1,
"controlled_failed_runs": 0,
},
provider_cost_rows=[],
window_start=end - timedelta(hours=24),
window_end=end,
)
assert scorecard["denominators"]["execution_observed_count"] == 1
assert scorecard["denominators"]["verifier_observed_count"] == 1
assert scorecard["denominators"]["shadow_or_no_write_execution_excluded_count"] == 1
corpus = scorecard["learning_corpus"]
assert corpus["status"] == ("learning_corpus_ready_same_run_verification_pending")
assert corpus["denominators"]["production_execution_observed_count"] == 1
assert corpus["denominators"]["verifier_observed_execution_count"] == 1
assert corpus["excluded_from_execution"] == {
"shadow_run_count": 1,
"non_shadow_no_step_run_count": 1,
"alert_count_with_shadow_or_no_write_execution_evidence": 1,
}
excluded_record = next(
row
for row in corpus["records"]
if row["outcomes"]["shadow_or_no_write_execution_excluded"]
)
assert excluded_record["outcomes"]["production_execution_observed"] is False
assert excluded_record["outcomes"]["verifier_observed"] is False
def test_incident_query_classifies_only_non_shadow_runtime_execution() -> None:
statement = " ".join(str(scorecard_module._INCIDENT_SAMPLES_SQL).split())
execution_events = statement.split("), execution_evidence AS (", 1)[1].split(
"AS execution_like_event", 1
)[0]
assert "AUTO_REPAIR_TRIGGERED" not in execution_events
assert (
"event_type IN ( 'EXECUTION_STARTED', 'EXECUTION_COMPLETED', "
"'CHANGE_APPLIED' )" in execution_events
)
assert "AS production_execution_evidence" in statement
assert "AS execution_exclusion_evidence" in statement
assert "context ->> 'is_shadow'" in statement
assert "context ->> 'execution_mode'" in statement
assert "context ->> 'side_effect_performed'" in statement
assert "'%no_write%'" in statement
assert "context ->> 'execution_mode', 'production'" not in statement
assert "context ->> 'execution_kind', 'executable'" not in statement
assert "bool_or(production_execution_event) AS execution_observed" in statement
assert "context ? 'post_verifier_passed'" in statement
def test_run_query_excludes_shadow_and_no_step_runs_from_controlled_outcomes() -> None:
@@ -182,8 +313,7 @@ def test_run_query_excludes_shadow_and_no_step_runs_from_controlled_outcomes() -
)
assert (
"WHERE is_shadow IS FALSE AND step_count > 0 "
"AND state IN ('failed', 'cancelled', 'timeout')"
in statement
"AND state IN ('failed', 'cancelled', 'timeout')" in statement
)

10
apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py
View File

@@ -155,8 +155,8 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
"active_or_completed_commitments": 68,
"by_status": {
"analysis_or_governance_complete": 6,
"source_implemented_runtime_pending": 35,
"in_progress": 26,
"source_implemented_runtime_pending": 36,
"in_progress": 25,
"not_started_or_no_current_evidence": 1,
"superseded": 2,
},
@@ -206,6 +206,12 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
assert "independent exit-code and freshness receipts" in " ".join(
commitments["AIA-CONV-041"]["source_evidence"]
)
assert commitments["AIA-CONV-045"]["status"] == (
"source_implemented_runtime_pending"
)
assert "deterministically recomputable" in " ".join(
commitments["AIA-CONV-045"]["source_evidence"]
)
assert "Host112" in commitments["AIA-CONV-049"]["title"]
assert commitments["AIA-CONV-049"]["status"] == (
"source_implemented_runtime_pending"