wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity

test(sre): add bounded Holmes shadow replay scorecard

Browse Source
This commit is contained in:
Your Name
2026-07-19 02:19:43 +08:00
parent 370f954622
commit a105f9470a
5 changed files with 933 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json
View File

@@ -657,9 +657,11 @@
"source_refs": [
"apps/api/src/services/pre_decision_investigator.py",
"apps/api/src/services/holmes_shadow_investigator.py",
"apps/api/src/services/holmes_shadow_replay.py",
"apps/api/src/services/evidence_snapshot.py",
"apps/api/src/core/config.py",
"apps/api/tests/test_holmes_shadow_investigator.py"
"apps/api/tests/test_holmes_shadow_investigator.py",
"apps/api/tests/test_holmes_shadow_replay.py"
],
"executor": "none in shadow",
"verifier": "offline replay quality and prompt-injection safety scorecard",
@@ -668,14 +670,17 @@
"confirmed_truth": [
"the optional PreDecisionInvestigator shadow adapter uses HolmesGPT /api/chat with a strict JSON schema and bounded timeout",
"evidence and returned text are sanitized, protected API key and endpoint are absent from receipts, and only an exact allowlisted internal origin whose runtime artifact header matches the pinned digest can be contacted",
"every trusted post-generation response must include prompt-token, completion-token and cost headers; missing or invalid usage metadata fails closed and the replay scorecard aggregates the bounded totals",
"tool calls, follow-up actions, invalid structured output and any runtime-authority claim are rejected fail closed",
"the advisory receipt merges without replacing existing diagnosis signals and is summarized into the persisted evidence text",
"the sanitized receipt is reused only through the existing 30-second evidence fingerprint cache and is marked as reused, preventing duplicate model calls for an identical cached evidence slice"
"the sanitized receipt is reused only through the existing 30-second evidence fingerprint cache and is marked as reused, preventing duplicate model calls for an identical cached evidence slice",
"the offline replay evaluator reuses AWOOOI candidate-visible inputs, keeps evaluation labels outside the model request, caps one batch at 50 sequential calls and persists only bounded metrics plus result digests",
"the replay scorecard reports contract, RCA-label, prompt-injection, action-surface and runtime-authority observations but cannot authorize canary promotion or runtime writes"
],
"runtime_gaps": [
"no approved internal immutable HolmesGPT artifact, allowlisted endpoint, modelList alias, offline replay scorecard, shadow canary or production receipt exists in this worktree"
"no approved internal immutable HolmesGPT artifact, allowlisted endpoint, modelList alias, runtime usage-header receipt, executed 50-record offline replay scorecard, shadow canary or production receipt exists in this worktree"
],
"next_action": "supply HolmesGPT from an approved internal immutable mirror, configure its protected key/modelList alias/exact allowlist/digest, then run offline replay and prompt-injection scorecards before any canary"
"next_action": "supply HolmesGPT from an approved internal immutable mirror, configure its protected key/modelList alias/exact allowlist/digest, then execute the bounded 50-record replay against sanitized historical fixtures before any canary"
},
{
"id": "AIA-SRE-013",