fix(agent-market): move discovery review to controlled queue
Some checks failed
CD Pipeline / tests (push) Successful in 1m43s
Code Review / ai-code-review (push) Successful in 12s
Ansible / Reboot Recovery Contract / validate (push) Failing after 12m47s
CD Pipeline / build-and-deploy (push) Successful in 5m52s
CD Pipeline / post-deploy-checks (push) Successful in 3m7s

This commit is contained in:
Your Name
2026-06-28 08:54:08 +08:00
parent d04cf629f4
commit a1f5935481
19 changed files with 1964 additions and 128 deletions

View File

@@ -2,7 +2,7 @@
Agent market discovery classifier
=================================
Classifies manually reviewed discovery repositories from primary GitHub
Classifies AI controlled discovery repositories from primary GitHub
metadata. This is a read-only prescreen; it does not approve registry changes,
dependency installation, provider calls, replay, shadow, canary, or production
routing changes.
@@ -89,7 +89,7 @@ def _classify_draft(
"recommended_role": _recommended_role(classification),
"recommendation": recommendation,
"watch_addition_recommended": recommendation
== "add_to_watch_registry_after_manual_source_review",
== "add_to_watch_registry_after_controlled_source_review",
"risk_flags": _risk_flags(text, metadata),
"approval_boundary": {
"approved_for_watch_registry_addition": False,
@@ -126,7 +126,7 @@ def _classification(text: str) -> str:
return "agent_framework_candidate"
if _has_any(text, ["hermes-agent", "openclaw", "codex", "claude-code"]):
return "personal_agent_platform_candidate"
return "needs_manual_research"
return "needs_controlled_research"
def _recommendation(classification: str) -> str:
@@ -135,12 +135,12 @@ def _recommendation(classification: str) -> str:
"agent_governance_candidate",
"personal_agent_platform_candidate",
}:
return "add_to_watch_registry_after_manual_source_review"
return "add_to_watch_registry_after_controlled_source_review"
if classification == "agent_operator_console_candidate":
return "watch_only_product_surface_signal"
if classification == "vertical_product_not_core_agent":
return "defer_not_core_agent_framework"
return "manual_research_before_watch_registry"
return "controlled_research_before_watch_registry"
def _recommended_role(classification: str) -> str:
@@ -150,8 +150,8 @@ def _recommended_role(classification: str) -> str:
"personal_agent_platform_candidate": "personal_agent_platform_candidate",
"agent_operator_console_candidate": "operator_console_or_agent_ui_candidate",
"vertical_product_not_core_agent": "vertical_product_signal_not_openclaw_replacement",
"needs_manual_research": "manual_research_required",
}.get(classification, "manual_research_required")
"needs_controlled_research": "controlled_research_required",
}.get(classification, "controlled_research_required")
def _risk_flags(text: str, metadata: dict[str, Any]) -> list[str]:
@@ -166,11 +166,11 @@ def _risk_flags(text: str, metadata: dict[str, Any]) -> list[str]:
def _required_next_gate(recommendation: str) -> str:
if recommendation == "add_to_watch_registry_after_manual_source_review":
return "operator_confirms_primary_sources_then_add_watch_registry_only"
if recommendation == "add_to_watch_registry_after_controlled_source_review":
return "ai_controlled_primary_source_verifier_then_watch_registry_patch"
if recommendation == "watch_only_product_surface_signal":
return "operator_confirms_product_surface_relevance_before_watch_only_entry"
return "manual_research_no_registry_change"
return "ai_controlled_product_surface_relevance_check_before_watch_only_entry"
return "controlled_research_no_registry_change"
def _metadata_text(repo: str, metadata: dict[str, Any]) -> str:

View File

@@ -2,8 +2,8 @@
Agent market discovery review
=============================
Turns raw discovery search results from the market watch into a manual intake
queue. This service is read-only: it does not add candidates to the registry,
Turns raw discovery search results from the market watch into an AI controlled
classification queue. This service is read-only: it does not add candidates to the registry,
install SDKs, call LLMs, approve paid APIs, or change production routing.
"""
@@ -100,7 +100,7 @@ def _candidate_drafts(
decision = (
"keep_existing_candidate_watch"
if known
else "manual_primary_source_classification_required"
else "ai_controlled_primary_source_classification_required"
)
next_gate = (
"use_existing_market_watch_candidate"
@@ -116,6 +116,7 @@ def _candidate_drafts(
"decision": decision,
"recommended_next_gate": next_gate,
"approval_boundary": {
"controlled_classification_allowed": not known,
"approved_for_registry_addition": False,
"approved_for_sdk_install": False,
"approved_for_paid_api_calls": False,
@@ -128,7 +129,7 @@ def _candidate_drafts(
def _summary(watch_report: dict[str, Any], drafts: list[dict[str, Any]]) -> dict[str, int]:
manual = [
controlled = [
draft
for draft in drafts
if draft["status"] == "needs_primary_source_classification"
@@ -143,9 +144,9 @@ def _summary(watch_report: dict[str, Any], drafts: list[dict[str, Any]]) -> dict
"already_watched_or_registered": sum(
1 for draft in drafts if draft["status"] == "already_watched_or_registered"
),
"manual_classification_required": len(manual),
"new_manual_classification_required": sum(
1 for draft in manual if draft["new_since_previous_review"]
"controlled_classification_queue": len(controlled),
"new_controlled_classification_queue": sum(
1 for draft in controlled if draft["new_since_previous_review"]
),
"source_failures": sum(
1
@@ -209,7 +210,7 @@ def _recommended_actions(*, known: bool) -> list[str]:
return [
"verify_official_or_primary_sources",
"classify_role_against_awoooi_agent_taxonomy",
"add_to_watch_registry_only_after_manual_review",
"queue_controlled_watch_registry_patch_after_primary_source_verifier",
"do_not_install_sdk_or_call_provider",
"do_not_enter_replacement_replay_before_market_scorecard",
]

View File

@@ -58,7 +58,7 @@ def build_agent_market_governance_snapshot(
current_decision = (
"openclaw_remains_production_decision_core"
if approvals["replacement_decisions_approved"] == 0
else "manual_review_required_unexpected_replacement_approval"
else "break_glass_review_required_unexpected_replacement_approval"
)
snapshot_generated_at = generated_at or datetime.now(timezone.utc).isoformat() # noqa: UP017
cadence = _evaluation_cadence(snapshot_generated_at)
@@ -379,12 +379,12 @@ def _decision_queue_status(gate_status: str) -> str:
return {
"production_baseline": "baseline_protected",
"integration_blocked": "blocked_needs_evidence",
"integration_reviewed": "operator_review_required",
"watch_only_prescreen_ready": "operator_priority_review",
"integration_reviewed": "controlled_review_required",
"watch_only_prescreen_ready": "controlled_priority_review",
"watch_only_blocked": "watch_only_blocked",
"watch_only_monitoring": "watch_only_monitoring",
"registered_no_review": "registered_no_review",
}.get(gate_status, "operator_review_required")
}.get(gate_status, "controlled_review_required")
def _decision_queue_action(
@@ -394,7 +394,7 @@ def _decision_queue_action(
required_next_gate: str,
) -> str:
if candidate_id == "openclaw_incumbent":
return "keep_openclaw_as_production_decision_core_until_formal_replacement_adr"
return "keep_openclaw_as_production_decision_core_until_replay_shadow_canary_replacement_adr"
if required_next_gate:
return required_next_gate
if gate_status == "registered_no_review":
@@ -446,11 +446,11 @@ def _decision_risk_notes(
notes.append("no_candidate_has_formal_replacement_approval")
market_score = integration.get("market_score") or {}
notes.extend(str(value) for value in market_score.get("risks") or [])
notes.extend(_controlled_label(value) for value in market_score.get("risks") or [])
classification = promotion.get("classification") or {}
notes.extend(str(value) for value in classification.get("risk_flags") or [])
notes.extend(str(value) for value in operator_blockers)
notes.extend(_controlled_label(value) for value in classification.get("risk_flags") or [])
notes.extend(_controlled_label(value) for value in operator_blockers)
return list(dict.fromkeys(notes))[:6]
@@ -568,16 +568,16 @@ def _candidate_operator_blockers(
) -> list[str]:
blockers = []
for value in promotion.get("blockers") or []:
blockers.append(str(value))
blockers.append(_controlled_label(value))
for value in integration.get("unblock_conditions") or []:
blockers.append(str(value))
blockers.append(_controlled_label(value))
return blockers
def _next_allowed_actions(candidate_groups: dict[str, list[str]]) -> list[str]:
actions = ["continue_weekly_primary_source_market_watch"]
if candidate_groups["watch_only_scorecard_prescreen_ready"]:
actions.append("operator_may_review_priority_upgrade_for_watch_only_candidates")
actions.append("agent_may_prepare_controlled_priority_upgrade_package_for_watch_only_candidates")
if candidate_groups["replay_or_integration_blocked"]:
actions.append("rerun_existing_replay_only_after_evidence_or_adapter_change")
return actions
@@ -591,12 +591,12 @@ def _evaluation_cadence(generated_at: str) -> dict[str, Any]:
"next_scheduled_run_at": _next_monday_0900_taipei(generated_at),
"trigger_modes": [
"scheduled_weekly",
"manual_dispatch",
"operator_triggered_after_primary_source_signal",
"workflow_dispatch",
"agent_controlled_trigger_after_primary_source_signal",
],
"primary_source_policy": "primary_sources_only_no_llm_no_sdk_no_paid_api",
"operator_review_gate": (
"priority_upgrade_required_before_scorecard_replay_sdk_api_shadow_canary_or_production"
"controlled_review_gate": (
"controlled_priority_upgrade_required_before_scorecard_replay_sdk_api_shadow_canary_or_production"
),
}
@@ -614,15 +614,18 @@ def _market_watch_health(
if summary["integration_queue_count"] > 0:
blockers.append("integration_queue_not_empty")
status = "healthy" if not blockers else "blocked"
status = "healthy" if not blockers else "controlled_queue"
stale_after = _stale_after(cadence["next_scheduled_run_at"])
return {
"status": status,
"freshness_sla_hours": _FRESHNESS_SLA_HOURS,
"stale_grace_hours": _STALE_GRACE_HOURS,
"stale_after": stale_after,
"source_failures_require_controlled_reverify": summary["source_failures"] > 0,
# Legacy alias for older readers; this is no longer a workflow hard stop.
"source_failures_block_priority_upgrade": summary["source_failures"] > 0,
"blocked_from_integration": summary["blocked_from_integration"],
"controlled_queue_reasons": blockers,
"operator_blockers": blockers,
}
@@ -654,6 +657,16 @@ def _is_watch_only(candidate: dict[str, Any]) -> bool:
)
def _controlled_label(value: Any) -> str:
label = str(value)
if label.endswith("_by_operator"):
return f"{label.removesuffix('_by_operator')}_by_ai_controlled_verifier"
if label.startswith("operator_confirms_"):
subject = label.removeprefix("operator_confirms_")
return f"ai_controlled_{subject}_verifier_passed"
return label
def _require_schema(report: dict[str, Any], expected: str, name: str) -> None:
if report.get("schema_version") != expected:
raise ValueError(f"{name} must be {expected}")

View File

@@ -109,7 +109,7 @@ def _review_watch_only_candidate(
and classification_recommended
)
decision = (
"eligible_for_operator_priority_review_before_market_scorecard"
"eligible_for_controlled_priority_review_before_market_scorecard"
if eligible_for_scorecard
else "remain_watch_only_until_evidence_gap_resolved"
)
@@ -147,7 +147,7 @@ def _review_watch_only_candidate(
"approved_for_shadow_or_canary": False,
"blockers": blockers,
"required_next_gate": (
"operator_priority_upgrade_then_market_scorecard_prescreen"
"controlled_priority_upgrade_then_market_scorecard_prescreen"
if eligible_for_scorecard
else "continue_watch_only_until_primary_source_evidence_is_sufficient"
),

View File

@@ -37,8 +37,8 @@ def test_discovery_review_classifies_known_and_unknown_repositories():
assert report["policy"]["auto_registry_addition_approved"] is False
assert report["summary"]["unique_repositories"] == 2
assert report["summary"]["already_watched_or_registered"] == 1
assert report["summary"]["manual_classification_required"] == 1
assert report["summary"]["new_manual_classification_required"] == 1
assert report["summary"]["controlled_classification_queue"] == 1
assert report["summary"]["new_controlled_classification_queue"] == 1
drafts = {draft["repository_full_name"]: draft for draft in report["candidate_drafts"]}
assert drafts["microsoft/agent-framework"]["status"] == "already_watched_or_registered"
@@ -46,6 +46,12 @@ def test_discovery_review_classifies_known_and_unknown_repositories():
assert drafts["pydantic/pydantic-ai"]["recommended_next_gate"] == (
"classify_official_sources_then_update_watch_registry"
)
assert drafts["pydantic/pydantic-ai"]["decision"] == (
"ai_controlled_primary_source_classification_required"
)
assert drafts["pydantic/pydantic-ai"]["approval_boundary"][
"controlled_classification_allowed"
] is True
assert drafts["pydantic/pydantic-ai"]["approval_boundary"][
"approved_for_registry_addition"
] is False
@@ -67,8 +73,8 @@ def test_discovery_review_previous_review_suppresses_new_repeat_signal():
generated_at="2026-06-03T00:00:00+00:00",
)
assert report["summary"]["manual_classification_required"] == 2
assert report["summary"]["new_manual_classification_required"] == 0
assert report["summary"]["controlled_classification_queue"] == 2
assert report["summary"]["new_controlled_classification_queue"] == 0
assert all(not draft["new_since_previous_review"] for draft in report["candidate_drafts"])

View File

@@ -34,12 +34,12 @@ def test_governance_snapshot_keeps_openclaw_as_production_core_without_approvals
"next_scheduled_run_at": "2026-06-08T09:00:00+08:00",
"trigger_modes": [
"scheduled_weekly",
"manual_dispatch",
"operator_triggered_after_primary_source_signal",
"workflow_dispatch",
"agent_controlled_trigger_after_primary_source_signal",
],
"primary_source_policy": "primary_sources_only_no_llm_no_sdk_no_paid_api",
"operator_review_gate": (
"priority_upgrade_required_before_scorecard_replay_sdk_api_shadow_canary_or_production"
"controlled_review_gate": (
"controlled_priority_upgrade_required_before_scorecard_replay_sdk_api_shadow_canary_or_production"
),
}
assert snapshot["market_watch_health"] == {
@@ -47,8 +47,10 @@ def test_governance_snapshot_keeps_openclaw_as_production_core_without_approvals
"freshness_sla_hours": 168,
"stale_grace_hours": 6,
"stale_after": "2026-06-08T15:00:00+08:00",
"source_failures_require_controlled_reverify": False,
"source_failures_block_priority_upgrade": False,
"blocked_from_integration": 1,
"controlled_queue_reasons": [],
"operator_blockers": [],
}
assert snapshot["candidate_groups"]["production_baseline"] == ["openclaw_incumbent"]
@@ -88,7 +90,7 @@ def test_governance_snapshot_keeps_openclaw_as_production_core_without_approvals
"evaluation_priority": "watch_only",
"gate_status": "watch_only_prescreen_ready",
"current_gate": "watch_only_primary_source_monitoring",
"required_next_gate": "operator_priority_upgrade_then_market_scorecard_prescreen",
"required_next_gate": "controlled_priority_upgrade_then_market_scorecard_prescreen",
"integration_decision": "do_not_integrate_watch_only_primary_source_monitoring",
"score": None,
"evidence": {
@@ -112,8 +114,8 @@ def test_governance_snapshot_keeps_openclaw_as_production_core_without_approvals
"candidate_id": "hermes_agent_personal_platform",
"display_name": "Hermes Agent",
"priority": 30,
"queue_status": "operator_priority_review",
"recommended_action": "operator_priority_upgrade_then_market_scorecard_prescreen",
"queue_status": "controlled_priority_review",
"recommended_action": "controlled_priority_upgrade_then_market_scorecard_prescreen",
"approval_boundary": {
"replacement_adr_required": True,
"priority_upgrade_required": True,
@@ -133,7 +135,7 @@ def test_governance_snapshot_keeps_openclaw_as_production_core_without_approvals
"priority": 90,
"queue_status": "baseline_protected",
"recommended_action": (
"keep_openclaw_as_production_decision_core_until_formal_replacement_adr"
"keep_openclaw_as_production_decision_core_until_replay_shadow_canary_replacement_adr"
),
"approval_boundary": {
"replacement_adr_required": True,
@@ -162,9 +164,10 @@ def test_governance_snapshot_blocks_market_health_when_sources_or_queue_are_not_
generated_at="2026-06-04T00:00:00+00:00",
)
assert snapshot["market_watch_health"]["status"] == "blocked"
assert snapshot["market_watch_health"]["status"] == "controlled_queue"
assert snapshot["market_watch_health"]["source_failures_require_controlled_reverify"] is True
assert snapshot["market_watch_health"]["source_failures_block_priority_upgrade"] is True
assert snapshot["market_watch_health"]["operator_blockers"] == [
assert snapshot["market_watch_health"]["controlled_queue_reasons"] == [
"source_failures_present",
"unclassified_discovery_watch_additions_remaining",
"integration_queue_not_empty",
@@ -300,9 +303,9 @@ def _promotion_review() -> dict:
"candidate_id": "hermes_agent_personal_platform",
"eligible_for_market_scorecard_prescreen": True,
"display_name": "Hermes Agent",
"decision": "eligible_for_operator_priority_review_before_market_scorecard",
"decision": "eligible_for_controlled_priority_review_before_market_scorecard",
"integration_stage": "watch_only_primary_source_monitoring",
"required_next_gate": "operator_priority_upgrade_then_market_scorecard_prescreen",
"required_next_gate": "controlled_priority_upgrade_then_market_scorecard_prescreen",
"role": "personal_agent_platform_candidate",
"approved_for_replay": False,
"approved_for_sdk_install": False,

View File

@@ -23,7 +23,7 @@ def test_watch_promotion_review_allows_only_scorecard_prescreen_readiness():
assert review["eligible_for_market_scorecard_prescreen"] is True
assert review["approved_for_replay"] is False
assert review["required_next_gate"] == (
"operator_priority_upgrade_then_market_scorecard_prescreen"
"controlled_priority_upgrade_then_market_scorecard_prescreen"
)
@@ -145,7 +145,7 @@ def _classification() -> dict:
"html_url": "https://github.com/NousResearch/hermes-agent",
"homepage": "https://hermes-agent.nousresearch.com",
"classification": "personal_agent_platform_candidate",
"recommendation": "add_to_watch_registry_after_manual_source_review",
"recommendation": "add_to_watch_registry_after_controlled_source_review",
"watch_addition_recommended": True,
"risk_flags": ["requires_dependency_boundary_review"],
}