Merge remote-tracking branch 'origin/main' into codex/p0-obs-002-20260715
This commit is contained in:
ogt
@@ -31,6 +31,7 @@ python3 ops/runner/guard-gitea-runner-pressure.py --root "$ROOT_DIR"
|
||||
echo "== Shell 語法 =="
|
||||
bash -n \
|
||||
scripts/reboot-recovery/full-stack-cold-start-check.sh \
|
||||
scripts/reboot-recovery/host188-edge-services-recover.sh \
|
||||
scripts/reboot-recovery/full-stack-recovery-scorecard.sh \
|
||||
scripts/reboot-recovery/dr-offsite-operator-checklist.sh \
|
||||
scripts/reboot-recovery/wait-dr-offsite-ready.sh \
|
||||
|
||||
@@ -86,7 +86,7 @@ deploy_to_host() {
|
||||
|
||||
AWOOOI_API_URL=https://awoooi.wooo.work
|
||||
TELEGRAM_BOT_TOKEN=CHANGE_ME
|
||||
SRE_GROUP_CHAT_ID=-1003711974679
|
||||
SRE_GROUP_CHAT_ID=CHANGE_ME
|
||||
SEND_COOLDOWN_SECONDS=300
|
||||
ACTION_COOLDOWN_SECONDS=300
|
||||
NOTIFY_COOLDOWN_SECONDS=1800
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
"""Bounded Docker disk-pressure cleanup for host reboot recovery.
|
||||
|
||||
This controller intentionally avoids `docker system prune`, volumes, containers,
|
||||
running images, databases, backups, and logs. It only removes dangling images
|
||||
that are not referenced by any container, and can optionally run a bounded
|
||||
BuildKit cache cleanup with an explicit keep-storage floor.
|
||||
running images, databases, backups, and logs. It removes only images that are
|
||||
not referenced by any container. Tagged-image cleanup is explicit, age-bounded,
|
||||
and preserves a per-repository rollback window. BuildKit cleanup is separately
|
||||
opt-in and preserves an explicit storage floor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -13,14 +14,18 @@ import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from typing import Any
|
||||
|
||||
UTC = timezone.utc # noqa: UP017 - controller still runs on Python 3.9 hosts.
|
||||
DEFAULT_MIN_AGE_HOURS = 24
|
||||
DEFAULT_KEEP_DANGLING_NEWEST = 20
|
||||
DEFAULT_TAGGED_MIN_AGE_HOURS = 168
|
||||
DEFAULT_KEEP_TAGGED_NEWEST_PER_REPOSITORY = 3
|
||||
DEFAULT_MAX_IMAGE_REMOVALS = 100
|
||||
DEFAULT_BUILDER_KEEP_STORAGE = "30GB"
|
||||
|
||||
|
||||
@@ -42,6 +47,30 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--host-label", default="")
|
||||
parser.add_argument("--min-age-hours", type=int, default=DEFAULT_MIN_AGE_HOURS)
|
||||
parser.add_argument("--keep-dangling-newest", type=int, default=DEFAULT_KEEP_DANGLING_NEWEST)
|
||||
parser.add_argument(
|
||||
"--include-tagged-images",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Also remove old, container-unreferenced tagged images while preserving "
|
||||
"the newest images for every repository."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tagged-min-age-hours",
|
||||
type=int,
|
||||
default=DEFAULT_TAGGED_MIN_AGE_HOURS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-tagged-newest-per-repository",
|
||||
type=int,
|
||||
default=DEFAULT_KEEP_TAGGED_NEWEST_PER_REPOSITORY,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-image-removals",
|
||||
type=int,
|
||||
default=DEFAULT_MAX_IMAGE_REMOVALS,
|
||||
help="Maximum image IDs removed by one controlled apply.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-dangling-images",
|
||||
action="store_true",
|
||||
@@ -102,8 +131,8 @@ def parse_docker_datetime(value: str) -> datetime:
|
||||
text = f"{head}.{frac_text}{suffix}"
|
||||
parsed = datetime.fromisoformat(text)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def chunked(values: list[str], size: int) -> Iterable[list[str]]:
|
||||
@@ -170,6 +199,45 @@ def get_dangling_images(docker_bin: str) -> list[ImageInfo]:
|
||||
return images
|
||||
|
||||
|
||||
def get_all_images(docker_bin: str) -> list[ImageInfo]:
|
||||
image_ids = {
|
||||
normalize_image_id(value)
|
||||
for value in docker(
|
||||
["image", "ls", "--all", "--quiet", "--no-trunc"],
|
||||
docker_bin,
|
||||
).stdout.split()
|
||||
if normalize_image_id(value)
|
||||
}
|
||||
images: list[ImageInfo] = []
|
||||
for group in chunked(sorted(image_ids), 100):
|
||||
result = docker(["image", "inspect", *group], docker_bin)
|
||||
payload = json.loads(result.stdout or "[]")
|
||||
for item in payload:
|
||||
image_id = normalize_image_id(str(item.get("Id") or ""))
|
||||
if not image_id:
|
||||
continue
|
||||
tags = item.get("RepoTags") or []
|
||||
images.append(
|
||||
ImageInfo(
|
||||
image_id=image_id,
|
||||
created_at=parse_docker_datetime(str(item.get("Created") or "")),
|
||||
size_bytes=int(item.get("Size") or 0),
|
||||
repo_tags=tuple(str(tag) for tag in tags if tag),
|
||||
)
|
||||
)
|
||||
return images
|
||||
|
||||
|
||||
def repository_from_tag(tag: str) -> str:
|
||||
value = str(tag or "").strip()
|
||||
if "@" in value:
|
||||
return value.split("@", 1)[0]
|
||||
tail = value.rsplit("/", 1)[-1]
|
||||
if ":" not in tail:
|
||||
return value
|
||||
return value.rsplit(":", 1)[0]
|
||||
|
||||
|
||||
def select_dangling_image_removals(
|
||||
images: list[ImageInfo],
|
||||
protected_ids: set[str],
|
||||
@@ -192,6 +260,45 @@ def select_dangling_image_removals(
|
||||
return sorted(dangling, key=lambda image: image.created_at)
|
||||
|
||||
|
||||
def select_tagged_image_removals(
|
||||
images: list[ImageInfo],
|
||||
protected_ids: set[str],
|
||||
*,
|
||||
now: datetime,
|
||||
min_age_hours: int,
|
||||
keep_newest_per_repository: int,
|
||||
) -> list[ImageInfo]:
|
||||
unreferenced = [
|
||||
image
|
||||
for image in images
|
||||
if normalize_image_id(image.image_id) not in protected_ids
|
||||
and image.repo_tags
|
||||
]
|
||||
repository_images: dict[str, list[ImageInfo]] = {}
|
||||
for image in unreferenced:
|
||||
for repository in {
|
||||
repository_from_tag(tag) for tag in image.repo_tags if tag
|
||||
}:
|
||||
repository_images.setdefault(repository, []).append(image)
|
||||
|
||||
retained_ids: set[str] = set()
|
||||
for repository_rows in repository_images.values():
|
||||
repository_rows.sort(key=lambda image: image.created_at, reverse=True)
|
||||
retained_ids.update(
|
||||
normalize_image_id(image.image_id)
|
||||
for image in repository_rows[:keep_newest_per_repository]
|
||||
)
|
||||
|
||||
cutoff_seconds = min_age_hours * 3600
|
||||
selected = [
|
||||
image
|
||||
for image in unreferenced
|
||||
if normalize_image_id(image.image_id) not in retained_ids
|
||||
and (now - image.created_at).total_seconds() >= cutoff_seconds
|
||||
]
|
||||
return sorted(selected, key=lambda image: image.created_at)
|
||||
|
||||
|
||||
def summarize_images(images: list[ImageInfo]) -> dict[str, Any]:
|
||||
return {
|
||||
"count": len(images),
|
||||
@@ -202,13 +309,29 @@ def summarize_images(images: list[ImageInfo]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def limit_image_removals(
|
||||
images: list[ImageInfo],
|
||||
*,
|
||||
max_removals: int,
|
||||
) -> list[ImageInfo]:
|
||||
return sorted(images, key=lambda image: image.created_at)[:max_removals]
|
||||
|
||||
|
||||
def remove_images(images: list[ImageInfo], docker_bin: str) -> list[str]:
|
||||
removed: list[str] = []
|
||||
for group in chunked([image.image_id for image in images], 25):
|
||||
if not group:
|
||||
continue
|
||||
dangling_ids = [image.image_id for image in images if not image.repo_tags]
|
||||
for group in chunked(dangling_ids, 25):
|
||||
docker(["image", "rm", *group], docker_bin)
|
||||
removed.extend(group)
|
||||
tagged_images = [image for image in images if image.repo_tags]
|
||||
tagged_references = [
|
||||
reference
|
||||
for image in tagged_images
|
||||
for reference in image.repo_tags
|
||||
]
|
||||
for group in chunked(tagged_references, 25):
|
||||
docker(["image", "rm", *group], docker_bin)
|
||||
removed.extend(image.image_id for image in tagged_images)
|
||||
return removed
|
||||
|
||||
|
||||
@@ -227,11 +350,11 @@ def builder_prune_command(args: argparse.Namespace) -> list[str]:
|
||||
|
||||
|
||||
def build_receipt(args: argparse.Namespace) -> dict[str, Any]:
|
||||
now = datetime.now(timezone.utc)
|
||||
now = datetime.now(UTC)
|
||||
before = current_disk_bytes(args.disk_path)
|
||||
protected_ids = get_container_image_ids(args.docker_bin)
|
||||
dangling_images = get_dangling_images(args.docker_bin)
|
||||
removal_candidates = (
|
||||
dangling_removal_candidates = (
|
||||
[]
|
||||
if args.skip_dangling_images
|
||||
else select_dangling_image_removals(
|
||||
@@ -242,6 +365,29 @@ def build_receipt(args: argparse.Namespace) -> dict[str, Any]:
|
||||
keep_newest=args.keep_dangling_newest,
|
||||
)
|
||||
)
|
||||
all_images = get_all_images(args.docker_bin) if args.include_tagged_images else []
|
||||
tagged_removal_candidates = (
|
||||
select_tagged_image_removals(
|
||||
all_images,
|
||||
protected_ids,
|
||||
now=now,
|
||||
min_age_hours=args.tagged_min_age_hours,
|
||||
keep_newest_per_repository=args.keep_tagged_newest_per_repository,
|
||||
)
|
||||
if args.include_tagged_images
|
||||
else []
|
||||
)
|
||||
candidates_by_id = {
|
||||
normalize_image_id(image.image_id): image
|
||||
for image in [*dangling_removal_candidates, *tagged_removal_candidates]
|
||||
}
|
||||
eligible_removal_candidates = sorted(
|
||||
candidates_by_id.values(), key=lambda image: image.created_at
|
||||
)
|
||||
removal_candidates = limit_image_removals(
|
||||
eligible_removal_candidates,
|
||||
max_removals=args.max_image_removals,
|
||||
)
|
||||
receipt: dict[str, Any] = {
|
||||
"schema_version": "awoooi_docker_disk_pressure_retention_cleanup_v1",
|
||||
"generated_at": now.isoformat(),
|
||||
@@ -254,12 +400,21 @@ def build_receipt(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"touches_databases": False,
|
||||
"touches_backups": False,
|
||||
"uses_docker_system_prune": False,
|
||||
"removes_only_unreferenced_dangling_images": True,
|
||||
"removes_only_container_unreferenced_images": True,
|
||||
"tagged_image_cleanup_requires_explicit_flag": True,
|
||||
"tagged_image_cleanup_preserves_repository_rollback_window": True,
|
||||
"single_apply_image_removal_is_count_bounded": True,
|
||||
"builder_cache_cleanup_requires_explicit_flag": True,
|
||||
},
|
||||
"parameters": {
|
||||
"min_age_hours": args.min_age_hours,
|
||||
"keep_dangling_newest": args.keep_dangling_newest,
|
||||
"include_tagged_images": args.include_tagged_images,
|
||||
"tagged_min_age_hours": args.tagged_min_age_hours,
|
||||
"keep_tagged_newest_per_repository": (
|
||||
args.keep_tagged_newest_per_repository
|
||||
),
|
||||
"max_image_removals": args.max_image_removals,
|
||||
"include_builder_cache": args.include_builder_cache,
|
||||
"builder_keep_storage": args.builder_keep_storage,
|
||||
"skip_dangling_images": args.skip_dangling_images,
|
||||
@@ -267,7 +422,12 @@ def build_receipt(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"disk_before": before,
|
||||
"protected_container_image_count": len(protected_ids),
|
||||
"dangling_image_total_count": len(dangling_images),
|
||||
"dangling_image_removal_plan": summarize_images(removal_candidates),
|
||||
"dangling_image_removal_plan": summarize_images(dangling_removal_candidates),
|
||||
"tagged_image_removal_plan": summarize_images(tagged_removal_candidates),
|
||||
"eligible_image_removal_plan": summarize_images(
|
||||
eligible_removal_candidates
|
||||
),
|
||||
"total_image_removal_plan": summarize_images(removal_candidates),
|
||||
"builder_cache_command": builder_prune_command(args)[1:] if args.include_builder_cache else None,
|
||||
"removed_image_ids": [],
|
||||
"builder_cache_cleanup_executed": False,
|
||||
@@ -292,6 +452,15 @@ def main() -> int:
|
||||
if args.keep_dangling_newest < 0:
|
||||
print("keep-dangling-newest must be >= 0", file=sys.stderr)
|
||||
return 2
|
||||
if args.tagged_min_age_hours <= 0:
|
||||
print("tagged-min-age-hours must be > 0", file=sys.stderr)
|
||||
return 2
|
||||
if args.keep_tagged_newest_per_repository < 1:
|
||||
print("keep-tagged-newest-per-repository must be >= 1", file=sys.stderr)
|
||||
return 2
|
||||
if args.max_image_removals < 1:
|
||||
print("max-image-removals must be >= 1", file=sys.stderr)
|
||||
return 2
|
||||
receipt = build_receipt(args)
|
||||
text = json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
if args.output:
|
||||
|
||||
@@ -23,7 +23,7 @@ if [[ -f "$SECRETS_FILE" ]]; then
|
||||
fi
|
||||
|
||||
: "${AWOOOI_API_URL:=https://awoooi.wooo.work}"
|
||||
: "${SRE_GROUP_CHAT_ID:=-1003711974679}"
|
||||
: "${SRE_GROUP_CHAT_ID:=}"
|
||||
: "${LOG_FILE:=/var/log/docker-health-monitor.log}"
|
||||
: "${SEND_COOLDOWN_SECONDS:=300}"
|
||||
: "${ACTION_COOLDOWN_SECONDS:=${SEND_COOLDOWN_SECONDS}}"
|
||||
|
||||
@@ -6,7 +6,6 @@ from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "ops" / "docker-disk-pressure-retention-cleanup.py"
|
||||
spec = importlib.util.spec_from_file_location("docker_disk_pressure_retention_cleanup", SCRIPT)
|
||||
@@ -14,6 +13,7 @@ module = importlib.util.module_from_spec(spec)
|
||||
assert spec and spec.loader
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
UTC = timezone.utc # noqa: UP017 - mirrors supported Python 3.9 runtime.
|
||||
|
||||
|
||||
def image(image_id: str, created_at: datetime, tags: tuple[str, ...] = ()):
|
||||
@@ -26,7 +26,7 @@ def image(image_id: str, created_at: datetime, tags: tuple[str, ...] = ()):
|
||||
|
||||
|
||||
def test_select_dangling_images_keeps_newest_and_protects_running_images() -> None:
|
||||
now = datetime(2026, 7, 1, 12, tzinfo=timezone.utc)
|
||||
now = datetime(2026, 7, 1, 12, tzinfo=UTC)
|
||||
images = [
|
||||
image("sha256:running", now - timedelta(hours=72)),
|
||||
image("oldest", now - timedelta(hours=72)),
|
||||
@@ -73,7 +73,7 @@ def test_parse_docker_datetime_accepts_nanosecond_fraction() -> None:
|
||||
|
||||
|
||||
def test_summary_never_reports_volumes_or_container_cleanup_boundary() -> None:
|
||||
now = datetime(2026, 7, 1, 12, tzinfo=timezone.utc)
|
||||
now = datetime(2026, 7, 1, 12, tzinfo=UTC)
|
||||
selected = module.select_dangling_image_removals(
|
||||
[image("old", now - timedelta(days=3))],
|
||||
set(),
|
||||
@@ -87,6 +87,88 @@ def test_summary_never_reports_volumes_or_container_cleanup_boundary() -> None:
|
||||
assert summary["sample_image_ids"] == ["old"]
|
||||
|
||||
|
||||
def test_tagged_cleanup_preserves_running_and_repository_rollback_window() -> None:
|
||||
now = datetime(2026, 7, 14, 12, tzinfo=UTC)
|
||||
images = [
|
||||
image("running", now - timedelta(days=30), ("registry:5000/app:running",)),
|
||||
image("latest", now - timedelta(days=1), ("registry:5000/app:latest",)),
|
||||
image("rollback-1", now - timedelta(days=2), ("registry:5000/app:sha-1",)),
|
||||
image("rollback-2", now - timedelta(days=3), ("registry:5000/app:sha-2",)),
|
||||
image("expired", now - timedelta(days=14), ("registry:5000/app:sha-old",)),
|
||||
image("other-repo", now - timedelta(days=14), ("registry:5000/other:only",)),
|
||||
]
|
||||
|
||||
selected = module.select_tagged_image_removals(
|
||||
images,
|
||||
{"running"},
|
||||
now=now,
|
||||
min_age_hours=7 * 24,
|
||||
keep_newest_per_repository=3,
|
||||
)
|
||||
|
||||
assert [item.image_id for item in selected] == ["expired"]
|
||||
|
||||
|
||||
def test_multi_repository_image_is_retained_when_any_repository_needs_it() -> None:
|
||||
now = datetime(2026, 7, 14, 12, tzinfo=UTC)
|
||||
shared = image(
|
||||
"shared",
|
||||
now - timedelta(days=30),
|
||||
("registry:5000/app:old", "registry:5000/critical:latest"),
|
||||
)
|
||||
selected = module.select_tagged_image_removals(
|
||||
[shared, image("app-new", now - timedelta(days=1), ("registry:5000/app:new",))],
|
||||
set(),
|
||||
now=now,
|
||||
min_age_hours=7 * 24,
|
||||
keep_newest_per_repository=1,
|
||||
)
|
||||
|
||||
assert selected == []
|
||||
|
||||
|
||||
def test_repository_parser_preserves_registry_port() -> None:
|
||||
assert module.repository_from_tag("registry:5000/team/app:sha") == "registry:5000/team/app"
|
||||
|
||||
|
||||
def test_single_apply_image_removal_is_count_bounded() -> None:
|
||||
now = datetime(2026, 7, 14, 12, tzinfo=UTC)
|
||||
candidates = [
|
||||
image(f"image-{index}", now - timedelta(days=index + 1))
|
||||
for index in range(5)
|
||||
]
|
||||
|
||||
selected = module.limit_image_removals(candidates, max_removals=2)
|
||||
|
||||
assert [item.image_id for item in selected] == ["image-4", "image-3"]
|
||||
|
||||
|
||||
def test_remove_images_batches_tagged_references(monkeypatch) -> None:
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def fake_docker(args: list[str], _docker_bin: str):
|
||||
calls.append(args)
|
||||
return SimpleNamespace(stdout="", stderr="", returncode=0)
|
||||
|
||||
monkeypatch.setattr(module, "docker", fake_docker)
|
||||
now = datetime(2026, 7, 14, 12, tzinfo=UTC)
|
||||
|
||||
removed = module.remove_images(
|
||||
[
|
||||
image("dangling", now),
|
||||
image("tagged-a", now, ("repo-a:old", "repo-b:old")),
|
||||
image("tagged-b", now, ("repo-c:old",)),
|
||||
],
|
||||
"docker",
|
||||
)
|
||||
|
||||
assert removed == ["dangling", "tagged-a", "tagged-b"]
|
||||
assert calls == [
|
||||
["image", "rm", "dangling"],
|
||||
["image", "rm", "repo-a:old", "repo-b:old", "repo-c:old"],
|
||||
]
|
||||
|
||||
|
||||
def test_cli_exposes_builder_cache_only_flag() -> None:
|
||||
help_text = module.run_command(["python3", str(SCRIPT), "--help"]).stdout
|
||||
|
||||
|
||||
@@ -231,7 +231,9 @@ probe_public_routes() {
|
||||
"https://stock.wooo.work/api/v1/system/freshness" \
|
||||
"https://gitea.wooo.work/" \
|
||||
"https://registry.wooo.work/v2/" \
|
||||
"https://harbor.wooo.work/"
|
||||
"https://harbor.wooo.work/" \
|
||||
"https://n8n.wooo.work/" \
|
||||
"https://ollama.wooo.work/"
|
||||
do
|
||||
code="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)"
|
||||
[ -n "$code" ] || code="000"
|
||||
|
||||
@@ -16,14 +16,19 @@ while [[ $# -gt 0 ]]; do
|
||||
esac
|
||||
done
|
||||
|
||||
: "${TELEGRAM_BOT_TOKEN:?TELEGRAM_BOT_TOKEN is required in host-local env}"
|
||||
: "${TELEGRAM_CHAT_ID:?TELEGRAM_CHAT_ID is required in host-local env}"
|
||||
|
||||
TEXT="[AWOOOI reboot-recovery] phase=${PHASE} status=${STATUS}
|
||||
${MESSAGE}
|
||||
artifact=${ARTIFACT_DIR}"
|
||||
|
||||
curl --fail --silent --show-error --max-time 10 \
|
||||
--request POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
--data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
|
||||
--data-urlencode "text=${TEXT}" >/dev/null
|
||||
# Reboot recovery is outside the canonical API receipt boundary. Do not read
|
||||
# a host-local Telegram binding and do not fall back to a direct provider call.
|
||||
# The orchestrator keeps its local evidence; a correlated gateway transport
|
||||
# must be supplied before Telegram delivery can be re-enabled.
|
||||
printf '%s\n' \
|
||||
"telegram_route_status=blocked_no_egress" \
|
||||
"telegram_provider_send_performed=false" \
|
||||
"telegram_blocked_reason=canonical_telegram_gateway_transport_required" \
|
||||
"telegram_message_class=reboot_recovery" \
|
||||
"telegram_phase=${PHASE}" \
|
||||
"telegram_status=${STATUS}" >&2
|
||||
exit 75
|
||||
|
||||
@@ -14,7 +14,7 @@ CHECK_TIMEOUT_SECONDS="${CHECK_TIMEOUT_SECONDS:-240}"
|
||||
CHECK_WATCH_INTERVAL_SECONDS="${CHECK_WATCH_INTERVAL_SECONDS:-10}"
|
||||
CHECK_WATCH_MAX_ATTEMPTS="${CHECK_WATCH_MAX_ATTEMPTS:-3}"
|
||||
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
|
||||
SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_120_121_188}"
|
||||
SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_112_120_121_188}"
|
||||
LOCK_FILE="${LOCK_FILE:-/tmp/awoooi-cold-start-textfile-exporter.lock}"
|
||||
|
||||
escape_label() {
|
||||
@@ -39,6 +39,8 @@ write_metric_file() {
|
||||
local public_route_tls_blocker="${15}"
|
||||
local host_120_unreachable_blocker="${16}"
|
||||
local backup_health_blocker="${17}"
|
||||
local host_112_unreachable_blocker="${18}"
|
||||
local host_112_guest_not_ready_blocker="${19}"
|
||||
local host scope
|
||||
host=$(escape_label "$HOST_LABEL")
|
||||
scope=$(escape_label "$SCOPE_LABEL")
|
||||
@@ -80,6 +82,8 @@ awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="k3s_node_fi
|
||||
awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="public_route_tls_failure",target="public_https"} $public_route_tls_blocker
|
||||
awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="host_unreachable",target="120"} $host_120_unreachable_blocker
|
||||
awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="backup_health_blocked",target="110"} $backup_health_blocker
|
||||
awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="host_unreachable",target="112"} $host_112_unreachable_blocker
|
||||
awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="guest_readiness_failed",target="112"} $host_112_guest_not_ready_blocker
|
||||
METRICS
|
||||
}
|
||||
|
||||
@@ -102,7 +106,7 @@ if [ ! -x "$CHECK_SCRIPT" ]; then
|
||||
tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX")
|
||||
last_green=$(cat "$state_file" 2>/dev/null || echo 0)
|
||||
printf 'CHECK_SCRIPT not executable: %s\n' "$CHECK_SCRIPT" >"$log_file"
|
||||
write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green" 0 0 0 0
|
||||
write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green" 0 0 0 0 0 0
|
||||
chmod 0644 "$tmp_metric"
|
||||
mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME"
|
||||
exit 0
|
||||
@@ -131,6 +135,8 @@ k3s_node_fs_blocker=0
|
||||
public_route_tls_blocker=0
|
||||
host_120_unreachable_blocker=0
|
||||
backup_health_blocker=0
|
||||
host_112_unreachable_blocker=0
|
||||
host_112_guest_not_ready_blocker=0
|
||||
|
||||
if [ -n "$summary_line" ]; then
|
||||
monitor_up=1
|
||||
@@ -166,6 +172,14 @@ if grep -Eq 'BLOCKED 110 backup health has stale expected jobs' "$log_file"; the
|
||||
backup_health_blocker=1
|
||||
fi
|
||||
|
||||
if grep -Eq 'BLOCKED (ping 192\.168\.0\.112|ssh port 192\.168\.0\.112:22|112 canonical guest readback unavailable)' "$log_file"; then
|
||||
host_112_unreachable_blocker=1
|
||||
fi
|
||||
|
||||
if grep -Eq 'BLOCKED 112 (canonical guest readback schema missing|systemd not running|guest readiness failed|console or required services not ready|Wazuh independent verifier failed|guest recovery timer not ready|cold-start probe write boundary violated)' "$log_file"; then
|
||||
host_112_guest_not_ready_blocker=1
|
||||
fi
|
||||
|
||||
end_ts=$(date +%s)
|
||||
if [ "$green" -eq 1 ]; then
|
||||
printf '%s\n' "$end_ts" >"$state_file"
|
||||
@@ -173,6 +187,6 @@ fi
|
||||
last_green=$(cat "$state_file" 2>/dev/null || echo 0)
|
||||
|
||||
tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX")
|
||||
write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green" "$k3s_node_fs_blocker" "$public_route_tls_blocker" "$host_120_unreachable_blocker" "$backup_health_blocker"
|
||||
write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green" "$k3s_node_fs_blocker" "$public_route_tls_blocker" "$host_120_unreachable_blocker" "$backup_health_blocker" "$host_112_unreachable_blocker" "$host_112_guest_not_ready_blocker"
|
||||
chmod 0644 "$tmp_metric"
|
||||
mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME"
|
||||
|
||||
@@ -246,6 +246,13 @@ registry_code_ready() {
|
||||
[ "$code" = "200" ] || [ "$code" = "401" ]
|
||||
}
|
||||
|
||||
readback_value() {
|
||||
local text="$1" key="$2" rest
|
||||
rest="${text#*${key}=}"
|
||||
[ "$rest" != "$text" ] || return 1
|
||||
printf '%s' "${rest%% *}"
|
||||
}
|
||||
|
||||
probe_tcp() {
|
||||
local host="$1"
|
||||
local port="$2"
|
||||
@@ -254,11 +261,11 @@ probe_tcp() {
|
||||
|
||||
print_neighbor_rows() {
|
||||
if command -v arp >/dev/null 2>&1; then
|
||||
arp -an | grep -E '192\.168\.0\.(110|120|121|188)'
|
||||
arp -an | grep -E '192\.168\.0\.(110|112|120|121|188)'
|
||||
return $?
|
||||
fi
|
||||
if command -v ip >/dev/null 2>&1; then
|
||||
ip neigh show | grep -E '192\.168\.0\.(110|120|121|188)'
|
||||
ip neigh show | grep -E '192\.168\.0\.(110|112|120|121|188)'
|
||||
return $?
|
||||
fi
|
||||
return 1
|
||||
@@ -267,14 +274,14 @@ print_neighbor_rows() {
|
||||
print_header() {
|
||||
echo "AWOOOI full-stack cold-start check"
|
||||
date '+%Y-%m-%d %H:%M:%S %Z'
|
||||
echo "Scope: 110 / 120 / 121 / 188. 112 Kali is intentionally skipped."
|
||||
echo "Scope: 110 / 112 / 120 / 121 / 188."
|
||||
echo "Baseline: ops/reboot-recovery/full-stack-cold-start-baseline.yml"
|
||||
}
|
||||
|
||||
check_network() {
|
||||
log_section "P0-NETWORK"
|
||||
local host
|
||||
for host in 110 120 121 188; do
|
||||
for host in 110 112 120 121 188; do
|
||||
if ping -c 1 -W 2 "192.168.0.$host" >/dev/null 2>&1; then
|
||||
ok "ping 192.168.0.$host"
|
||||
else
|
||||
@@ -297,6 +304,50 @@ check_network() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_112() {
|
||||
log_section "P0-112-SECURITY"
|
||||
local out schema systemd_state guest_ready console_ready services_ready
|
||||
local recovery_timer_ready manager_verifier_ready runtime_write_performed
|
||||
if ! out=$(host_cmd "kali@192.168.0.112" \
|
||||
'/usr/local/sbin/awoooi-host112-guest-readiness --check' 2>&1); then
|
||||
fail "112 canonical guest readback unavailable"
|
||||
echo "$out"
|
||||
return
|
||||
fi
|
||||
|
||||
schema="$(readback_value "$out" schema_version || true)"
|
||||
systemd_state="$(readback_value "$out" systemd_state || true)"
|
||||
guest_ready="$(readback_value "$out" guest_ready || true)"
|
||||
console_ready="$(readback_value "$out" console_ready || true)"
|
||||
services_ready="$(readback_value "$out" services_ready || true)"
|
||||
recovery_timer_ready="$(readback_value "$out" recovery_timer_ready || true)"
|
||||
manager_verifier_ready="$(readback_value "$out" manager_independent_verifier_ready || true)"
|
||||
runtime_write_performed="$(readback_value "$out" runtime_write_performed || true)"
|
||||
echo "HOST112_READBACK schema=${schema:-missing} systemd=${systemd_state:-missing} guest=${guest_ready:-missing} console=${console_ready:-missing} services=${services_ready:-missing} timer=${recovery_timer_ready:-missing} manager_verifier=${manager_verifier_ready:-missing} runtime_write=${runtime_write_performed:-missing}"
|
||||
|
||||
[ "$schema" = "host112_guest_recovery_v2" ] \
|
||||
&& ok "112 canonical guest readback schema verified" \
|
||||
|| fail "112 canonical guest readback schema missing"
|
||||
[ "$systemd_state" = "running" ] \
|
||||
&& ok "112 systemd running" \
|
||||
|| fail "112 systemd not running"
|
||||
[ "$guest_ready" = "1" ] \
|
||||
&& ok "112 guest readiness verified" \
|
||||
|| fail "112 guest readiness failed"
|
||||
[ "$console_ready" = "1" ] && [ "$services_ready" = "1" ] \
|
||||
&& ok "112 console and required services ready" \
|
||||
|| fail "112 console or required services not ready"
|
||||
[ "$manager_verifier_ready" = "1" ] \
|
||||
&& ok "112 Wazuh independent verifier ready" \
|
||||
|| fail "112 Wazuh independent verifier failed"
|
||||
[ "$recovery_timer_ready" = "1" ] \
|
||||
&& ok "112 guest recovery timer ready" \
|
||||
|| fail "112 guest recovery timer not ready"
|
||||
[ "$runtime_write_performed" = "0" ] \
|
||||
&& ok "112 cold-start probe remained read-only" \
|
||||
|| fail "112 cold-start probe write boundary violated"
|
||||
}
|
||||
|
||||
check_188() {
|
||||
log_section "P0-188-DATA"
|
||||
local out
|
||||
@@ -660,6 +711,8 @@ check_public_routes() {
|
||||
"langfuse|https://langfuse.wooo.work/"
|
||||
"bitan|https://bitan.wooo.work/"
|
||||
"aiops|https://aiops.wooo.work/"
|
||||
"n8n|https://n8n.wooo.work/"
|
||||
"open_webui|https://ollama.wooo.work/"
|
||||
)
|
||||
|
||||
for item in "${routes[@]}"; do
|
||||
@@ -1067,6 +1120,7 @@ fi
|
||||
|
||||
print_header
|
||||
check_network
|
||||
check_112
|
||||
check_188
|
||||
check_110
|
||||
check_k3s
|
||||
|
||||
288
scripts/reboot-recovery/host188-edge-services-recover.sh
Executable file
288
scripts/reboot-recovery/host188-edge-services-recover.sh
Executable file
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
MODE="check"
|
||||
TRACE_ID=""
|
||||
RUN_ID=""
|
||||
WORK_ITEM_ID=""
|
||||
|
||||
HOST_ID="192.168.0.188"
|
||||
N8N_IMAGE="harbor.wooo.work/dockerhub-cache/n8nio/n8n@sha256:dbe732489e5b8941aa89ca71b320ccb7f80319fcccd0f2c9e05f561562d804e5"
|
||||
N8N_IMAGE_ID="sha256:19c1ad26c0c285fddd7fc1300ff670b7977afebf52a21de4dfb2a5a2b2a75e2b"
|
||||
OPEN_WEBUI_IMAGE="harbor.wooo.work/ghcr-cache/open-webui/open-webui@sha256:a673af6e7fab29822a0d32d71184d3f0277347a693633c86372643590c921915"
|
||||
OPEN_WEBUI_IMAGE_ID="sha256:5bddf2d93bc307b99c1fde8b8d0c2dc3c563e0e24fc3eebb6488c539fa55d080"
|
||||
BACKUP_IMAGE="harbor.wooo.work/dockerhub-cache/library/alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"
|
||||
N8N_DIR="/home/ollama/n8n"
|
||||
N8N_OVERRIDE="${N8N_DIR}/docker-compose.agent99.yml"
|
||||
OPEN_WEBUI_DIR="/home/ollama/open-webui"
|
||||
OPEN_WEBUI_COMPOSE="${OPEN_WEBUI_DIR}/docker-compose.yml"
|
||||
RECEIPT_ROOT="/home/ollama/agent99-runtime-recovery"
|
||||
|
||||
usage() {
|
||||
printf 'usage: %s --check | --apply --trace-id ID --run-id ID --work-item-id ID\n' "$0" >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
valid_id() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9._:-]+$ ]]
|
||||
}
|
||||
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
--check)
|
||||
MODE="check"
|
||||
shift
|
||||
;;
|
||||
--apply)
|
||||
MODE="apply"
|
||||
shift
|
||||
;;
|
||||
--trace-id)
|
||||
(($# >= 2)) || usage
|
||||
TRACE_ID="$2"
|
||||
shift 2
|
||||
;;
|
||||
--run-id)
|
||||
(($# >= 2)) || usage
|
||||
RUN_ID="$2"
|
||||
shift 2
|
||||
;;
|
||||
--work-item-id)
|
||||
(($# >= 2)) || usage
|
||||
WORK_ITEM_ID="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$MODE" == "apply" ]]; then
|
||||
[[ -n "$TRACE_ID" && -n "$RUN_ID" && -n "$WORK_ITEM_ID" ]] || usage
|
||||
valid_id "$TRACE_ID" && valid_id "$RUN_ID" && valid_id "$WORK_ITEM_ID" || usage
|
||||
fi
|
||||
|
||||
bool_json() {
|
||||
if [[ "$1" == "true" ]]; then printf 'true'; else printf 'false'; fi
|
||||
}
|
||||
|
||||
http_status() {
|
||||
local url="$1"
|
||||
local resolve_arg="${2:-}"
|
||||
local status
|
||||
if [[ -n "$resolve_arg" ]]; then
|
||||
status="$(curl -k -sS -L -o /dev/null -w '%{http_code}' --resolve "$resolve_arg" --max-time 12 "$url" 2>/dev/null || true)"
|
||||
else
|
||||
status="$(curl -sS -L -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)"
|
||||
fi
|
||||
[[ "$status" =~ ^[0-9]{3}$ ]] || status="000"
|
||||
printf '%s' "$status"
|
||||
}
|
||||
|
||||
non_5xx() {
|
||||
local status="$1"
|
||||
[[ "$status" =~ ^[0-9]{3}$ ]] && ((10#$status >= 200 && 10#$status < 500))
|
||||
}
|
||||
|
||||
container_value() {
|
||||
local name="$1"
|
||||
local format="$2"
|
||||
docker inspect "$name" --format "$format" 2>/dev/null || true
|
||||
}
|
||||
|
||||
collect_state() {
|
||||
local runtime_write="${1:-false}"
|
||||
local terminal="${2:-observed}"
|
||||
local receipt_path="${3:-}"
|
||||
|
||||
local n8n_status n8n_image n8n_restart n8n_restarts n8n_local n8n_public
|
||||
local webui_status webui_health webui_image webui_restart webui_restarts webui_local webui_public
|
||||
local n8n_config_ok=false webui_config_ok=false n8n_ok=false webui_ok=false overall=false
|
||||
|
||||
n8n_status="$(container_value n8n '{{.State.Status}}')"
|
||||
n8n_image="$(container_value n8n '{{.Image}}')"
|
||||
n8n_restart="$(container_value n8n '{{.HostConfig.RestartPolicy.Name}}')"
|
||||
n8n_restarts="$(container_value n8n '{{.RestartCount}}')"
|
||||
[[ "$n8n_restarts" =~ ^[0-9]+$ ]] || n8n_restarts=0
|
||||
n8n_local="$(http_status 'http://127.0.0.1:5678/')"
|
||||
n8n_public="$(http_status 'https://n8n.wooo.work/' 'n8n.wooo.work:443:127.0.0.1')"
|
||||
|
||||
webui_status="$(container_value open-webui '{{.State.Status}}')"
|
||||
webui_health="$(container_value open-webui '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}')"
|
||||
webui_image="$(container_value open-webui '{{.Image}}')"
|
||||
webui_restart="$(container_value open-webui '{{.HostConfig.RestartPolicy.Name}}')"
|
||||
webui_restarts="$(container_value open-webui '{{.RestartCount}}')"
|
||||
[[ "$webui_restarts" =~ ^[0-9]+$ ]] || webui_restarts=0
|
||||
webui_local="$(http_status 'http://127.0.0.1:3010/')"
|
||||
webui_public="$(http_status 'https://ollama.wooo.work/' 'ollama.wooo.work:443:127.0.0.1')"
|
||||
|
||||
if [[ -f "$N8N_OVERRIDE" ]] && grep -Fqx " image: ${N8N_IMAGE}" "$N8N_OVERRIDE"; then
|
||||
n8n_config_ok=true
|
||||
fi
|
||||
if [[ -f "$OPEN_WEBUI_COMPOSE" ]] && grep -Fqx " image: ${OPEN_WEBUI_IMAGE}" "$OPEN_WEBUI_COMPOSE"; then
|
||||
webui_config_ok=true
|
||||
fi
|
||||
|
||||
if [[ "$n8n_config_ok" == true && "$n8n_status" == running && "$n8n_image" == "$N8N_IMAGE_ID" && "$n8n_restart" == always ]] && non_5xx "$n8n_local" && non_5xx "$n8n_public"; then
|
||||
n8n_ok=true
|
||||
fi
|
||||
if [[ "$webui_config_ok" == true && "$webui_status" == running && "$webui_health" == healthy && "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" && "$webui_restart" == always ]] && non_5xx "$webui_local" && non_5xx "$webui_public"; then
|
||||
webui_ok=true
|
||||
fi
|
||||
if [[ "$n8n_ok" == true && "$webui_ok" == true ]]; then
|
||||
overall=true
|
||||
fi
|
||||
|
||||
printf '{"schemaVersion":"agent99_host188_edge_services_v1","mode":"%s","traceId":"%s","runId":"%s","workItemId":"%s","host":"%s","overallHealthy":%s,"runtimeWritePerformed":%s,"secretValueRead":false,"rawDataRead":false,"terminal":"%s","receiptPath":"%s","services":[{"name":"n8n","publicUrl":"https://n8n.wooo.work","containerStatus":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s},{"name":"open-webui","publicUrl":"https://ollama.wooo.work","containerStatus":"%s","health":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s}],"rollback":{"automatic":false,"snapshotPreserved":true,"procedure":"stop_changed_containers_then_restore_run_scoped_snapshot_after_owner_break_glass"},"prohibitedActions":["host_reboot","vm_power_change","firewall_change","database_content_read","secret_read","external_registry_direct_pull"]}\n' \
|
||||
"$MODE" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$HOST_ID" \
|
||||
"$(bool_json "$overall")" "$(bool_json "$runtime_write")" "$terminal" "$receipt_path" \
|
||||
"$n8n_status" "$(bool_json "$([[ "$n8n_image" == "$N8N_IMAGE_ID" ]] && echo true || echo false)")" "$n8n_restart" "$n8n_restarts" "$n8n_local" "$n8n_public" "$(bool_json "$n8n_config_ok")" "$(bool_json "$n8n_ok")" \
|
||||
"$webui_status" "$webui_health" "$(bool_json "$([[ "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" ]] && echo true || echo false)")" "$webui_restart" "$webui_restarts" "$webui_local" "$webui_public" "$(bool_json "$webui_config_ok")" "$(bool_json "$webui_ok")"
|
||||
|
||||
[[ "$overall" == true ]]
|
||||
}
|
||||
|
||||
if [[ "$MODE" == "check" ]]; then
|
||||
if collect_state false observed ""; then
|
||||
exit 0
|
||||
fi
|
||||
exit 2
|
||||
fi
|
||||
|
||||
exec 9>/tmp/agent99-host188-edge-services.lock
|
||||
if ! flock -w 30 9; then
|
||||
collect_state false single_flight_lock_timeout "" || true
|
||||
exit 75
|
||||
fi
|
||||
|
||||
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
||||
OPERATION_LOG="${RECEIPT_DIR}/operation.log"
|
||||
RECEIPT_PATH="${RECEIPT_DIR}/receipt.json"
|
||||
|
||||
emit_apply_failure() {
|
||||
local exit_code="$1"
|
||||
local result
|
||||
trap - ERR
|
||||
set +e
|
||||
result="$(collect_state true "apply_failed_exit_${exit_code}" "$RECEIPT_PATH" || true)"
|
||||
printf '%s\n' "$result" | tee "$RECEIPT_PATH"
|
||||
chmod 600 "$RECEIPT_PATH" 2>/dev/null || true
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
mkdir -p "$RECEIPT_DIR" "$N8N_DIR" "$OPEN_WEBUI_DIR"
|
||||
chmod 700 "$RECEIPT_DIR"
|
||||
: >"$OPERATION_LOG"
|
||||
chmod 600 "$OPERATION_LOG"
|
||||
trap 'emit_apply_failure $?' ERR
|
||||
|
||||
for image in "$BACKUP_IMAGE" "$N8N_IMAGE" "$OPEN_WEBUI_IMAGE"; do
|
||||
timeout 1200 docker pull "$image" >>"$OPERATION_LOG" 2>&1
|
||||
done
|
||||
|
||||
snapshot_with_helper() {
|
||||
local source_path="$1"
|
||||
local archive_name="$2"
|
||||
if [[ -s "${RECEIPT_DIR}/${archive_name}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
docker run --rm --pull never --userns=host \
|
||||
-v "${source_path}:/data:ro" -v "${RECEIPT_DIR}:/backup" \
|
||||
"$BACKUP_IMAGE" sh -c "umask 077; tar czf /backup/${archive_name} -C /data ." \
|
||||
>>"$OPERATION_LOG" 2>&1
|
||||
}
|
||||
|
||||
[[ -d "${N8N_DIR}/data" && -f "${N8N_DIR}/docker-compose.yml" ]] || {
|
||||
collect_state false n8n_base_assets_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true
|
||||
exit 3
|
||||
}
|
||||
|
||||
if ! docker volume inspect open-webui >/dev/null 2>&1; then
|
||||
docker volume create open-webui >>"$OPERATION_LOG"
|
||||
fi
|
||||
|
||||
snapshot_with_helper "${N8N_DIR}/data" "n8n-data-before.tgz"
|
||||
docker run --rm --pull never --userns=host \
|
||||
-v open-webui:/data:ro -v "${RECEIPT_DIR}:/backup" \
|
||||
"$BACKUP_IMAGE" sh -c 'umask 077; tar czf /backup/open-webui-volume-before.tgz -C /data .' \
|
||||
>>"$OPERATION_LOG" 2>&1
|
||||
docker run --rm --pull never --userns=host -v "${RECEIPT_DIR}:/backup" "$BACKUP_IMAGE" \
|
||||
chown -R "$(id -u):$(id -g)" /backup >>"$OPERATION_LOG" 2>&1
|
||||
chmod 600 "${RECEIPT_DIR}/n8n-data-before.tgz" "${RECEIPT_DIR}/open-webui-volume-before.tgz"
|
||||
|
||||
tmp_n8n="$(mktemp "${N8N_OVERRIDE}.XXXXXX")"
|
||||
cat >"$tmp_n8n" <<EOF
|
||||
services:
|
||||
n8n:
|
||||
image: ${N8N_IMAGE}
|
||||
EOF
|
||||
chmod 600 "$tmp_n8n"
|
||||
mv "$tmp_n8n" "$N8N_OVERRIDE"
|
||||
|
||||
tmp_webui="$(mktemp "${OPEN_WEBUI_COMPOSE}.XXXXXX")"
|
||||
cat >"$tmp_webui" <<EOF
|
||||
services:
|
||||
open-webui:
|
||||
image: ${OPEN_WEBUI_IMAGE}
|
||||
container_name: open-webui
|
||||
restart: always
|
||||
ports:
|
||||
- "3010:8080"
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
environment:
|
||||
OLLAMA_BASE_URL: http://host.docker.internal:11434
|
||||
OLLAMA_BASE_URLS: http://host.docker.internal:11434
|
||||
volumes:
|
||||
- open-webui:/app/backend/data
|
||||
|
||||
volumes:
|
||||
open-webui:
|
||||
external: true
|
||||
EOF
|
||||
chmod 600 "$tmp_webui"
|
||||
mv "$tmp_webui" "$OPEN_WEBUI_COMPOSE"
|
||||
|
||||
remap_uid="$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subuid 2>/dev/null || true)"
|
||||
remap_gid="$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subgid 2>/dev/null || true)"
|
||||
if [[ "$remap_uid" =~ ^[0-9]+$ && "$remap_gid" =~ ^[0-9]+$ ]]; then
|
||||
node_uid=$((remap_uid + 1000))
|
||||
node_gid=$((remap_gid + 1000))
|
||||
docker run --rm --pull never --userns=host -v "${N8N_DIR}/data:/data" "$BACKUP_IMAGE" \
|
||||
chown -R "${node_uid}:${node_gid}" /data >>"$OPERATION_LOG" 2>&1
|
||||
else
|
||||
collect_state true docker_userns_mapping_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true
|
||||
exit 4
|
||||
fi
|
||||
|
||||
docker compose --project-directory "$N8N_DIR" \
|
||||
-f "${N8N_DIR}/docker-compose.yml" -f "$N8N_OVERRIDE" \
|
||||
up -d --pull never >>"$OPERATION_LOG" 2>&1
|
||||
webui_project="$(docker inspect open-webui --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || true)"
|
||||
if [[ -n "$(docker ps -aq --filter name='^/open-webui$')" && "$webui_project" != "open-webui" ]]; then
|
||||
docker rm -f open-webui >>"$OPERATION_LOG" 2>&1
|
||||
fi
|
||||
docker compose --project-directory "$OPEN_WEBUI_DIR" -f "$OPEN_WEBUI_COMPOSE" \
|
||||
up -d --pull never >>"$OPERATION_LOG" 2>&1
|
||||
|
||||
deadline=$((SECONDS + 360))
|
||||
while ((SECONDS < deadline)); do
|
||||
if collect_state true verifying "$RECEIPT_PATH" >/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
if result="$(collect_state true deployed_verified "$RECEIPT_PATH")"; then
|
||||
trap - ERR
|
||||
printf '%s\n' "$result" | tee "$RECEIPT_PATH"
|
||||
chmod 600 "$RECEIPT_PATH"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
result="$(collect_state true verifier_failed_snapshot_preserved "$RECEIPT_PATH" || true)"
|
||||
trap - ERR
|
||||
printf '%s\n' "$result" | tee "$RECEIPT_PATH"
|
||||
chmod 600 "$RECEIPT_PATH"
|
||||
exit 5
|
||||
@@ -63,6 +63,31 @@ def test_full_stack_cold_start_check_bounds_ssh_probes() -> None:
|
||||
assert "SSH_110_RECOVERY_PACKAGE_NEXT_ACTION verify_or_preinstall_local_recovery_package_from_console_before_harbor_repair_retry" in text
|
||||
|
||||
|
||||
def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None:
|
||||
text = COLD_START_CHECK.read_text(encoding="utf-8")
|
||||
baseline = (ROOT / "ops" / "reboot-recovery" / "full-stack-cold-start-baseline.yml").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
exporter = (ROOT / "scripts" / "reboot-recovery" / "cold-start-textfile-exporter.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
deploy_verifier = VERIFY_DEPLOY.read_text(encoding="utf-8")
|
||||
|
||||
assert "Scope: 110 / 112 / 120 / 121 / 188." in text
|
||||
assert "for host in 110 112 120 121 188; do" in text
|
||||
assert 'log_section "P0-112-SECURITY"' in text
|
||||
assert "awoooi-host112-guest-readiness --check" in text
|
||||
assert 'check_112\ncheck_188' in text
|
||||
assert '"112": "Kali security, Wazuh, guest readiness and bounded recovery"' in baseline
|
||||
assert "- id: P0-112-SECURITY" in baseline
|
||||
assert "112 Kali is intentionally skipped" not in text
|
||||
assert "not part of cold-start release gate" not in baseline
|
||||
assert 'SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_112_120_121_188}"' in exporter
|
||||
assert 'reason="host_unreachable",target="112"' in exporter
|
||||
assert 'reason="guest_readiness_failed",target="112"' in exporter
|
||||
assert 'scope="110_112_120_121_188"' in deploy_verifier
|
||||
|
||||
|
||||
def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None:
|
||||
text = COLD_START_CHECK.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def read(relative: str) -> str:
|
||||
return (ROOT / relative).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_agent99_config_covers_host188_edge_services_and_public_routes() -> None:
|
||||
for name in ("agent99.config.example.json", "agent99.config.99.example.json"):
|
||||
config = json.loads(read(name))
|
||||
edge = config["edgeServiceRecovery"]
|
||||
|
||||
assert edge["enabled"] is True
|
||||
assert edge["host"] == "192.168.0.188"
|
||||
assert edge["scriptPath"] == "/home/ollama/bin/agent99-host188-edge-services-recover.sh"
|
||||
assert set(edge["requiredServices"]) == {"n8n", "open-webui"}
|
||||
assert "https://n8n.wooo.work" in config["publicUrls"]
|
||||
assert "https://ollama.wooo.work" in config["publicUrls"]
|
||||
|
||||
|
||||
def test_host188_recovery_uses_only_internal_pinned_images() -> None:
|
||||
script = read("scripts/reboot-recovery/host188-edge-services-recover.sh")
|
||||
|
||||
assert "harbor.wooo.work/dockerhub-cache/n8nio/n8n@sha256:" in script
|
||||
assert "harbor.wooo.work/ghcr-cache/open-webui/open-webui@sha256:" in script
|
||||
assert "harbor.wooo.work/dockerhub-cache/library/alpine@sha256:" in script
|
||||
assert "ghcr.io/" not in script
|
||||
assert "docker.n8n.io/" not in script
|
||||
assert ":latest" not in script
|
||||
assert "external_registry_direct_pull" in script
|
||||
|
||||
|
||||
def test_host188_check_mode_is_read_only_and_apply_is_bounded() -> None:
|
||||
script = read("scripts/reboot-recovery/host188-edge-services-recover.sh")
|
||||
check_block = script.split('if [[ "$MODE" == "check" ]]', 1)[1].split(
|
||||
"exec 9>", 1
|
||||
)[0]
|
||||
|
||||
for forbidden in (
|
||||
"docker pull",
|
||||
"docker run",
|
||||
"docker compose",
|
||||
"mkdir",
|
||||
"mv ",
|
||||
"chown",
|
||||
"chmod",
|
||||
"tee ",
|
||||
):
|
||||
assert forbidden not in check_block
|
||||
|
||||
assert "flock -w 30" in script
|
||||
assert "snapshot_with_helper" in script
|
||||
assert "--pull never" in script
|
||||
assert '"secretValueRead":false' in script
|
||||
assert '"rawDataRead":false' in script
|
||||
assert '"automatic":false' in script
|
||||
assert "host_reboot" in script
|
||||
assert "vm_power_change" in script
|
||||
|
||||
|
||||
def test_agent99_recover_binds_check_apply_verify_and_completion() -> None:
|
||||
control = read("agent99-control-plane.ps1")
|
||||
deploy = read("agent99-deploy.ps1")
|
||||
playbook = read("infra/ansible/playbooks/188-ai-web.yml")
|
||||
|
||||
assert "function Invoke-AgentHost188EdgeRecovery" in control
|
||||
assert 'New-AgentRecoveryPhase "host188_edge_services"' in control
|
||||
assert 'New-AgentOutcomeCheck "host188_edge_services_ready"' in control
|
||||
assert "$host188EdgeRecovery.runtimeWritePerformed" in control
|
||||
assert "$host188EdgeRecovery.verified" in control
|
||||
assert "container_local_upstream_and_public_https" in control
|
||||
assert 'name = "canonical_public_route_coverage"' in deploy
|
||||
assert "agent99-host188-edge-services-recover.sh --check" in playbook
|
||||
assert "--work-item-id AIA-P0-006-EDGE-502-188" in playbook
|
||||
assert "not ansible_check_mode" in playbook
|
||||
@@ -129,10 +129,15 @@ require_remote_pattern \
|
||||
"110 deployed check script carries SSH host-key policy"
|
||||
|
||||
require_remote_pattern \
|
||||
'awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188",mode="read_only"} 1' \
|
||||
'awoooi_cold_start_monitor_up{host="110",scope="110_112_120_121_188",mode="read_only"} 1' \
|
||||
"/home/wooo/node_exporter_textfiles/cold_start_recovery.prom" \
|
||||
"110 cold-start monitor produced parseable metrics"
|
||||
|
||||
require_remote_pattern \
|
||||
'awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="guest_readiness_failed",target="112"}' \
|
||||
"/home/wooo/node_exporter_textfiles/cold_start_recovery.prom" \
|
||||
"110 cold-start monitor exports host112 readiness blocker series"
|
||||
|
||||
report_runtime_state
|
||||
report_cold_start_alerts
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ Bot API、不讀 secret、不連線主機，也不啟動任何 runtime gate。
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -164,11 +165,44 @@ def require_contains(label: str, text: str, marker: str) -> None:
|
||||
raise SystemExit(f"BLOCKED {label}: missing {marker!r}")
|
||||
|
||||
|
||||
def function_segment(text: str, marker: str, *, limit: int = 2200) -> str:
|
||||
start = text.find(marker)
|
||||
if start == -1:
|
||||
raise SystemExit(f"BLOCKED source function marker missing: {marker!r}")
|
||||
return text[start : start + limit]
|
||||
def function_segment(text: str, marker: str) -> str:
|
||||
"""Return the complete named function using Python AST source boundaries."""
|
||||
marker_prefix, separator, function_name = marker.partition("def ")
|
||||
function_name = function_name.strip()
|
||||
if not separator or not function_name.isidentifier():
|
||||
raise SystemExit(f"BLOCKED invalid source function marker: {marker!r}")
|
||||
|
||||
expected_type: type[ast.FunctionDef] | type[ast.AsyncFunctionDef]
|
||||
expected_type = (
|
||||
ast.AsyncFunctionDef
|
||||
if marker_prefix.strip() == "async"
|
||||
else ast.FunctionDef
|
||||
)
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError as exc:
|
||||
raise SystemExit(
|
||||
f"BLOCKED telegram gateway source is not valid Python: {exc.msg}"
|
||||
) from exc
|
||||
|
||||
matches = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, expected_type) and node.name == function_name
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit(
|
||||
"BLOCKED source function marker must resolve exactly once: "
|
||||
f"{marker!r}, matches={len(matches)}"
|
||||
)
|
||||
|
||||
node = matches[0]
|
||||
if node.end_lineno is None:
|
||||
raise SystemExit(
|
||||
f"BLOCKED source function boundary unavailable: {marker!r}"
|
||||
)
|
||||
lines = text.splitlines(keepends=True)
|
||||
return "".join(lines[node.lineno - 1 : node.end_lineno])
|
||||
|
||||
|
||||
def git_commit(root: Path) -> str:
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a repo-only Telegram notification egress inventory.
|
||||
|
||||
This scanner identifies Telegram Bot API sendMessage paths that can bypass
|
||||
TelegramGateway's final-exit formatter. It does not read secrets, call
|
||||
Telegram, modify workflows, or send notifications.
|
||||
This scanner identifies guarded Telegram Bot API paths that can bypass
|
||||
TelegramGateway's final-exit formatter. Local ``.github/workflows`` files are
|
||||
audited only as frozen legacy source truth. It does not read secrets, call
|
||||
Telegram, modify or execute workflows, or send notifications.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
@@ -22,21 +24,102 @@ from typing import Any
|
||||
TAIPEI = timezone(timedelta(hours=8))
|
||||
|
||||
SCAN_ROOTS = (
|
||||
Path("agent99-control-plane.ps1"),
|
||||
Path(".gitea/workflows"),
|
||||
Path(".github/workflows"),
|
||||
Path("scripts/ops"),
|
||||
Path("scripts/ci"),
|
||||
Path("scripts/reboot-recovery"),
|
||||
Path("apps/api/src"),
|
||||
)
|
||||
SCAN_SUFFIXES = {".py", ".sh", ".js", ".yml", ".yaml"}
|
||||
SCAN_SUFFIXES = {".py", ".ps1", ".sh", ".js", ".yml", ".yaml"}
|
||||
|
||||
DIRECT_BOT_API_RE = re.compile(
|
||||
r"api\.telegram\.org/bot.*sendMessage|sendMessage.*api\.telegram\.org/bot"
|
||||
GUARDED_BOT_METHODS = (
|
||||
"sendMessage",
|
||||
"sendDocument",
|
||||
"sendPhoto",
|
||||
"sendMediaGroup",
|
||||
"editMessageText",
|
||||
"sendAnimation",
|
||||
"sendVideo",
|
||||
"sendAudio",
|
||||
"sendVoice",
|
||||
)
|
||||
BOT_TOKEN_URL_RE = re.compile(
|
||||
r"api\.telegram\.org/bot.*?/(?P<method>"
|
||||
+ "|".join(re.escape(method) for method in GUARDED_BOT_METHODS)
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
GUARDED_BOT_METHOD_RE = re.compile(
|
||||
r"\b(?P<method>"
|
||||
+ "|".join(re.escape(method) for method in GUARDED_BOT_METHODS)
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
COMPACT_BOT_ENDPOINT_RE = re.compile(
|
||||
r"api\.telegram\.org.{0,1024}?bot.{0,512}?/(?P<method>"
|
||||
+ "|".join(re.escape(method) for method in GUARDED_BOT_METHODS)
|
||||
+ r")",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DIRECT_HTTP_TRANSPORT_RE = re.compile(
|
||||
r"(?:"
|
||||
r"requests\s*\.\s*Session\s*\([^)]*\)\s*\.\s*(?:post|request|send)\s*\(|"
|
||||
r"requests\s*\.\s*(?:post|request)\s*\(|"
|
||||
r"httpx\s*\.\s*(?:AsyncClient|Client)\s*\([^)]*\)\s*\.\s*(?:post|request|send)\s*\(|"
|
||||
r"httpx\s*\.\s*(?:post|request)\s*\(|"
|
||||
r"(?:urllib\s*\.\s*request\s*\.\s*)?build_opener\s*\([^)]*\)\s*\.\s*open\s*\(|"
|
||||
r"urllib\s*\.\s*request\s*\.\s*(?:urlopen|Request)\s*\(|"
|
||||
r"\burlopen\s*\(|"
|
||||
r"\b(?:_?opener)\s*\.\s*open\s*\(|"
|
||||
r"\bInvoke-(?:WebRequest|RestMethod)\b|"
|
||||
r"\.\s*PostAsync\s*\(|"
|
||||
r"\.\s*Upload(?:String|Data|File)\s*\(|"
|
||||
r"\b(?:fetch|curl|wget)\b|"
|
||||
r"\b(?:_http_client|client|session)\s*\.\s*(?:post|request|send)\s*\("
|
||||
r")",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
TELEGRAM_CONTEXT_RE = re.compile(
|
||||
r"(?:telegram|bot[_-]?token|chat[_-]?id)", re.IGNORECASE
|
||||
)
|
||||
BOT_TOKEN_CONTEXT_RE = re.compile(
|
||||
r"(?:\bbot[_-]?token\b|\btelegram[_-]?(?:bot[_-]?)?token\b|\$Token\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
CHAT_CONTEXT_RE = re.compile(
|
||||
r"(?:\bchat[_-]?id\b|\$ChatId\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
POWERSHELL_FUNCTION_RE = re.compile(
|
||||
r"^function\s+(?P<name>[A-Za-z0-9_-]+)\s*\{", re.IGNORECASE | re.MULTILINE
|
||||
)
|
||||
CANONICAL_FINAL_EXITS = {
|
||||
(
|
||||
"apps/api/src/services/telegram_gateway.py",
|
||||
"TelegramGateway._send_request",
|
||||
),
|
||||
}
|
||||
_AST_IMPORT_NODE_TYPES = (ast.Import, ast.ImportFrom)
|
||||
_AST_FUNCTION_NODE_TYPES = (ast.FunctionDef, ast.AsyncFunctionDef)
|
||||
_AST_SEQUENCE_TARGET_NODE_TYPES = (ast.Tuple, ast.List)
|
||||
_AST_NON_MODULE_SCOPE_NODE_TYPES = (
|
||||
ast.ClassDef,
|
||||
ast.FunctionDef,
|
||||
ast.AsyncFunctionDef,
|
||||
ast.Lambda,
|
||||
)
|
||||
GATEWAY_CALLSITE_RE = re.compile(
|
||||
r"(?:send_alert_notification\(|\b(?:tg|gw|gateway|telegram)\.send_text\(|_send_request\(\s*[\"']sendMessage[\"'])"
|
||||
)
|
||||
SECRET_INTERPOLATION_RE = re.compile(r"\$\{\{\s*secrets\.[^}]+\}\}")
|
||||
BOT_TOKEN_URL_RE = re.compile(r"api\.telegram\.org/bot.*?/sendMessage")
|
||||
BOT_TOKEN_FRAGMENT_RE = re.compile(
|
||||
r"bot[^/\s\"']+/(?P<method>"
|
||||
+ "|".join(re.escape(method) for method in GUARDED_BOT_METHODS)
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
REQUIRED_OWNER_FIELDS = [
|
||||
"egress_surface_id",
|
||||
@@ -134,6 +217,10 @@ def iter_scannable_files(root: Path) -> list[Path]:
|
||||
absolute_root = root / scan_root
|
||||
if not absolute_root.exists():
|
||||
continue
|
||||
if absolute_root.is_file():
|
||||
if absolute_root.suffix in SCAN_SUFFIXES:
|
||||
files.append(absolute_root)
|
||||
continue
|
||||
for path in absolute_root.rglob("*"):
|
||||
if path.is_file() and path.suffix in SCAN_SUFFIXES:
|
||||
files.append(path)
|
||||
@@ -143,22 +230,454 @@ def iter_scannable_files(root: Path) -> list[Path]:
|
||||
def sanitize_excerpt(line: str) -> str:
|
||||
excerpt = line.strip()
|
||||
excerpt = SECRET_INTERPOLATION_RE.sub("${{ secrets.<redacted> }}", excerpt)
|
||||
excerpt = BOT_TOKEN_URL_RE.sub("api.telegram.org/bot<redacted>/sendMessage", excerpt)
|
||||
excerpt = BOT_TOKEN_URL_RE.sub(
|
||||
lambda match: f"api.telegram.org/bot<redacted>/{match.group('method')}",
|
||||
excerpt,
|
||||
)
|
||||
excerpt = BOT_TOKEN_FRAGMENT_RE.sub(
|
||||
lambda match: f"bot<redacted>/{match.group('method')}",
|
||||
excerpt,
|
||||
)
|
||||
return excerpt[:180]
|
||||
|
||||
|
||||
def _compact_source(source: str) -> str:
|
||||
without_string_prefixes = re.sub(
|
||||
r"(?i)\b(?:fr|rf|f|r|u|b)(?=[\"'])",
|
||||
"",
|
||||
source,
|
||||
)
|
||||
return re.sub(r"[\s\"'`+()]+", "", without_string_prefixes)
|
||||
|
||||
|
||||
_PYTHON_HTTP_TRANSPORT_CALL_RE = re.compile(
|
||||
r"^(?:"
|
||||
r"requests\.(?:post|request)|"
|
||||
r"requests\.Session\(\)\.(?:post|request|send)|"
|
||||
r"httpx\.(?:post|request)|"
|
||||
r"httpx\.(?:AsyncClient|Client)\(\)\.(?:post|request|send)|"
|
||||
r"urllib\.request\.(?:urlopen|Request)|"
|
||||
r"urllib\.request\.build_opener\(\)\.open"
|
||||
r")$"
|
||||
)
|
||||
|
||||
|
||||
def _record_python_import_alias(
|
||||
node: ast.AST,
|
||||
aliases: dict[str, str],
|
||||
) -> None:
|
||||
if isinstance(node, ast.Import):
|
||||
for imported in node.names:
|
||||
bound_name = imported.asname or imported.name.split(".", 1)[0]
|
||||
aliases[bound_name] = (
|
||||
imported.name if imported.asname else bound_name
|
||||
)
|
||||
return
|
||||
|
||||
module = str(node.module or "").strip(".")
|
||||
if not module:
|
||||
return
|
||||
for imported in node.names:
|
||||
if imported.name == "*":
|
||||
continue
|
||||
aliases[imported.asname or imported.name] = (
|
||||
f"{module}.{imported.name}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_python_expression(
|
||||
node: ast.AST,
|
||||
aliases: dict[str, str],
|
||||
) -> str | None:
|
||||
if isinstance(node, ast.Name):
|
||||
return aliases.get(node.id, node.id)
|
||||
if isinstance(node, ast.Attribute):
|
||||
owner = _resolve_python_expression(node.value, aliases)
|
||||
return f"{owner}.{node.attr}" if owner else None
|
||||
if isinstance(node, ast.Call):
|
||||
callable_name = _resolve_python_expression(node.func, aliases)
|
||||
return f"{callable_name}()" if callable_name else None
|
||||
return None
|
||||
|
||||
|
||||
def _assignment_target_names(node: ast.AST) -> list[str]:
|
||||
if isinstance(node, ast.Name):
|
||||
return [node.id]
|
||||
if isinstance(node, _AST_SEQUENCE_TARGET_NODE_TYPES):
|
||||
return [
|
||||
name
|
||||
for child in node.elts
|
||||
for name in _assignment_target_names(child)
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def _python_parent_map(tree: ast.AST) -> dict[ast.AST, ast.AST]:
|
||||
return {
|
||||
child: parent
|
||||
for parent in ast.walk(tree)
|
||||
for child in ast.iter_child_nodes(parent)
|
||||
}
|
||||
|
||||
|
||||
def _python_function_qualified_name(
|
||||
function: ast.FunctionDef | ast.AsyncFunctionDef,
|
||||
parents: dict[ast.AST, ast.AST],
|
||||
) -> str:
|
||||
parts = [function.name]
|
||||
parent = parents.get(function)
|
||||
while parent is not None:
|
||||
if isinstance(parent, (ast.ClassDef, *_AST_FUNCTION_NODE_TYPES)):
|
||||
parts.append(parent.name)
|
||||
parent = parents.get(parent)
|
||||
return ".".join(reversed(parts))
|
||||
|
||||
|
||||
def _nearest_python_function(
|
||||
node: ast.AST,
|
||||
parents: dict[ast.AST, ast.AST],
|
||||
) -> ast.FunctionDef | ast.AsyncFunctionDef | None:
|
||||
parent = parents.get(node)
|
||||
while parent is not None:
|
||||
if isinstance(parent, _AST_FUNCTION_NODE_TYPES):
|
||||
return parent
|
||||
parent = parents.get(parent)
|
||||
return None
|
||||
|
||||
|
||||
def _is_python_module_scope(
|
||||
node: ast.AST,
|
||||
parents: dict[ast.AST, ast.AST],
|
||||
) -> bool:
|
||||
parent = parents.get(node)
|
||||
while parent is not None:
|
||||
if isinstance(parent, _AST_NON_MODULE_SCOPE_NODE_TYPES):
|
||||
return False
|
||||
parent = parents.get(parent)
|
||||
return True
|
||||
|
||||
|
||||
def _resolve_python_transport_calls(
|
||||
nodes: list[ast.AST],
|
||||
aliases: dict[str, str],
|
||||
) -> list[tuple[int, str]]:
|
||||
resolved_calls: list[tuple[int, str]] = []
|
||||
for node in sorted(
|
||||
nodes,
|
||||
key=lambda item: (
|
||||
int(getattr(item, "lineno", 0)),
|
||||
int(getattr(item, "col_offset", 0)),
|
||||
0 if isinstance(item, _AST_IMPORT_NODE_TYPES) else 1,
|
||||
),
|
||||
):
|
||||
if isinstance(node, _AST_IMPORT_NODE_TYPES):
|
||||
_record_python_import_alias(node, aliases)
|
||||
continue
|
||||
if isinstance(node, ast.Assign):
|
||||
resolved_value = _resolve_python_expression(node.value, aliases)
|
||||
for target in node.targets:
|
||||
for name in _assignment_target_names(target):
|
||||
if resolved_value:
|
||||
aliases[name] = resolved_value
|
||||
else:
|
||||
aliases.pop(name, None)
|
||||
continue
|
||||
if isinstance(node, ast.AnnAssign):
|
||||
resolved_value = (
|
||||
_resolve_python_expression(node.value, aliases)
|
||||
if node.value is not None
|
||||
else None
|
||||
)
|
||||
for name in _assignment_target_names(node.target):
|
||||
if resolved_value:
|
||||
aliases[name] = resolved_value
|
||||
else:
|
||||
aliases.pop(name, None)
|
||||
continue
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
qualified_call = _resolve_python_expression(node.func, aliases)
|
||||
if qualified_call and _PYTHON_HTTP_TRANSPORT_CALL_RE.fullmatch(
|
||||
qualified_call
|
||||
):
|
||||
resolved_calls.append((node.lineno, qualified_call))
|
||||
return sorted(set(resolved_calls))
|
||||
|
||||
|
||||
def _python_transport_calls_by_scope(
|
||||
text: str,
|
||||
) -> tuple[
|
||||
dict[tuple[str, int, int], list[tuple[int, str]]],
|
||||
list[tuple[int, str]],
|
||||
]:
|
||||
"""Resolve HTTP aliases in function and true module-level AST scopes."""
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return {}, []
|
||||
|
||||
parents = _python_parent_map(tree)
|
||||
module_aliases: dict[str, str] = {}
|
||||
module_nodes = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if _is_python_module_scope(node, parents)
|
||||
]
|
||||
module_calls = _resolve_python_transport_calls(
|
||||
module_nodes,
|
||||
module_aliases,
|
||||
)
|
||||
|
||||
transports: dict[tuple[str, int, int], list[tuple[int, str]]] = {}
|
||||
function_nodes = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, _AST_FUNCTION_NODE_TYPES)
|
||||
and node.end_lineno is not None
|
||||
]
|
||||
for function in function_nodes:
|
||||
aliases = dict(module_aliases)
|
||||
arguments = [
|
||||
*function.args.posonlyargs,
|
||||
*function.args.args,
|
||||
*function.args.kwonlyargs,
|
||||
]
|
||||
if function.args.vararg is not None:
|
||||
arguments.append(function.args.vararg)
|
||||
if function.args.kwarg is not None:
|
||||
arguments.append(function.args.kwarg)
|
||||
for argument in arguments:
|
||||
aliases.pop(argument.arg, None)
|
||||
|
||||
scoped_nodes = [
|
||||
node
|
||||
for node in ast.walk(function)
|
||||
if node is function
|
||||
or _nearest_python_function(node, parents) is function
|
||||
]
|
||||
resolved_calls = _resolve_python_transport_calls(scoped_nodes, aliases)
|
||||
|
||||
if resolved_calls:
|
||||
qualified_name = _python_function_qualified_name(function, parents)
|
||||
transports[
|
||||
(
|
||||
qualified_name,
|
||||
function.lineno,
|
||||
int(function.end_lineno),
|
||||
)
|
||||
] = resolved_calls
|
||||
return transports, module_calls
|
||||
|
||||
|
||||
def _transport_window_units(
|
||||
text: str,
|
||||
*,
|
||||
excluded_line_ranges: list[tuple[int, int]],
|
||||
) -> list[tuple[str, str, int, int, str]]:
|
||||
lines = text.splitlines()
|
||||
units: list[tuple[str, str, int, int, str]] = []
|
||||
for match in DIRECT_HTTP_TRANSPORT_RE.finditer(text):
|
||||
line_number = text.count("\n", 0, match.start()) + 1
|
||||
if any(start <= line_number <= end for start, end in excluded_line_ranges):
|
||||
continue
|
||||
start_line = max(1, line_number - 20)
|
||||
end_line = min(len(lines), line_number + 20)
|
||||
units.append(
|
||||
(
|
||||
"<module>",
|
||||
"<module>",
|
||||
start_line,
|
||||
end_line,
|
||||
"\n".join(lines[start_line - 1 : end_line]),
|
||||
)
|
||||
)
|
||||
return units
|
||||
|
||||
|
||||
def _source_units(
|
||||
relative_path: str,
|
||||
text: str,
|
||||
) -> list[tuple[str, str, int, int, str]]:
|
||||
lines = text.splitlines()
|
||||
units: list[tuple[str, str, int, int, str]] = []
|
||||
ranges: list[tuple[int, int]] = []
|
||||
if relative_path.endswith(".py"):
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
tree = None
|
||||
if tree is not None:
|
||||
parents = _python_parent_map(tree)
|
||||
nodes = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, _AST_FUNCTION_NODE_TYPES)
|
||||
and getattr(node, "end_lineno", None)
|
||||
]
|
||||
for node in sorted(
|
||||
nodes, key=lambda item: (item.lineno, item.end_lineno or item.lineno)
|
||||
):
|
||||
end_line = int(node.end_lineno or node.lineno)
|
||||
ranges.append((node.lineno, end_line))
|
||||
units.append(
|
||||
(
|
||||
node.name,
|
||||
_python_function_qualified_name(node, parents),
|
||||
node.lineno,
|
||||
end_line,
|
||||
"\n".join(lines[node.lineno - 1 : end_line]),
|
||||
)
|
||||
)
|
||||
elif relative_path.endswith(".ps1"):
|
||||
matches = list(POWERSHELL_FUNCTION_RE.finditer(text))
|
||||
for index, match in enumerate(matches):
|
||||
start_line = text.count("\n", 0, match.start()) + 1
|
||||
end_offset = (
|
||||
matches[index + 1].start() if index + 1 < len(matches) else len(text)
|
||||
)
|
||||
end_line = text.count("\n", 0, end_offset) + 1
|
||||
ranges.append((start_line, end_line))
|
||||
units.append(
|
||||
(
|
||||
match.group("name"),
|
||||
match.group("name"),
|
||||
start_line,
|
||||
end_line,
|
||||
text[match.start() : end_offset],
|
||||
)
|
||||
)
|
||||
return units + _transport_window_units(text, excluded_line_ranges=ranges)
|
||||
|
||||
|
||||
def scan_direct_bot_api_surfaces(
|
||||
relative_path: str,
|
||||
text: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Find direct Telegram transports, including split URLs and custom senders."""
|
||||
|
||||
source_lines = text.splitlines()
|
||||
findings: list[dict[str, Any]] = []
|
||||
seen: set[tuple[int, str, str]] = set()
|
||||
python_transports, module_transports = (
|
||||
_python_transport_calls_by_scope(text)
|
||||
if relative_path.endswith(".py")
|
||||
else ({}, [])
|
||||
)
|
||||
source_units = _source_units(relative_path, text)
|
||||
for line_number, _qualified_call in module_transports:
|
||||
start_line = max(1, line_number - 20)
|
||||
end_line = min(len(source_lines), line_number + 20)
|
||||
source_units.append(
|
||||
(
|
||||
"<module>",
|
||||
"<module>",
|
||||
start_line,
|
||||
end_line,
|
||||
"\n".join(source_lines[start_line - 1 : end_line]),
|
||||
)
|
||||
)
|
||||
|
||||
for (
|
||||
function_name,
|
||||
function_qualified,
|
||||
start_line,
|
||||
_end_line,
|
||||
source,
|
||||
) in source_units:
|
||||
if (relative_path, function_qualified) in CANONICAL_FINAL_EXITS:
|
||||
continue
|
||||
resolved_python_transports = (
|
||||
[
|
||||
item
|
||||
for item in module_transports
|
||||
if start_line <= item[0] <= _end_line
|
||||
]
|
||||
if function_qualified == "<module>"
|
||||
else python_transports.get(
|
||||
(function_qualified, start_line, _end_line),
|
||||
[],
|
||||
)
|
||||
)
|
||||
transport_match = DIRECT_HTTP_TRANSPORT_RE.search(source)
|
||||
if transport_match is None and not resolved_python_transports:
|
||||
continue
|
||||
method_match = GUARDED_BOT_METHOD_RE.search(source)
|
||||
endpoint_match = COMPACT_BOT_ENDPOINT_RE.search(_compact_source(source))
|
||||
telegram_named = bool(
|
||||
re.search(r"(?:telegram|bot)", function_name, re.IGNORECASE)
|
||||
)
|
||||
telegram_context = bool(TELEGRAM_CONTEXT_RE.search(source))
|
||||
token_and_chat_context = bool(
|
||||
BOT_TOKEN_CONTEXT_RE.search(source) and CHAT_CONTEXT_RE.search(source)
|
||||
)
|
||||
if (
|
||||
endpoint_match is None
|
||||
and method_match is None
|
||||
and not ((telegram_named and telegram_context) or token_and_chat_context)
|
||||
):
|
||||
continue
|
||||
|
||||
method = (
|
||||
endpoint_match.group("method")
|
||||
if endpoint_match is not None
|
||||
else method_match.group("method")
|
||||
if method_match is not None
|
||||
else "dynamic"
|
||||
)
|
||||
line_number = (
|
||||
resolved_python_transports[0][0]
|
||||
if resolved_python_transports
|
||||
else start_line + source[: transport_match.start()].count("\n")
|
||||
)
|
||||
detection_kind = (
|
||||
"direct_bot_api_endpoint"
|
||||
if endpoint_match is not None
|
||||
else "custom_direct_sender"
|
||||
)
|
||||
key = (line_number, method.lower(), detection_kind)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
source_line = (
|
||||
source_lines[line_number - 1] if line_number <= len(source_lines) else ""
|
||||
)
|
||||
findings.append(
|
||||
{
|
||||
"line": line_number,
|
||||
"method": method,
|
||||
"detection_kind": detection_kind,
|
||||
"function": function_name,
|
||||
"function_qualified": function_qualified,
|
||||
"sanitized_excerpt": sanitize_excerpt(source_line),
|
||||
}
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def surface_kind(relative_path: str) -> str:
|
||||
if relative_path.startswith(".github/workflows/"):
|
||||
return "github_frozen_legacy_workflow_direct_bot_api"
|
||||
if relative_path.startswith(".gitea/workflows/"):
|
||||
return "gitea_workflow_direct_bot_api"
|
||||
if relative_path.startswith("scripts/ops/"):
|
||||
return "ops_script_direct_bot_api"
|
||||
if relative_path.startswith("scripts/ci/"):
|
||||
return "ci_script_direct_bot_api"
|
||||
if relative_path == "agent99-control-plane.ps1":
|
||||
return "agent99_control_plane_direct_bot_api"
|
||||
if relative_path.startswith("scripts/reboot-recovery/"):
|
||||
return "reboot_recovery_direct_bot_api"
|
||||
if relative_path.startswith("apps/api/src/"):
|
||||
return "api_direct_bot_api"
|
||||
return "other_direct_bot_api"
|
||||
|
||||
|
||||
def source_truth_classification(relative_path: str) -> str:
|
||||
if relative_path.startswith(".github/workflows/"):
|
||||
return "frozen_legacy_source_truth"
|
||||
return "active_repo_source_truth"
|
||||
|
||||
|
||||
def line_hash(relative_path: str, line_number: int, line: str) -> str:
|
||||
payload = f"{relative_path}:{line_number}:{line.strip()}".encode("utf-8")
|
||||
return hashlib.sha256(payload).hexdigest()[:16]
|
||||
@@ -173,40 +692,50 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
for path in files:
|
||||
relative_path = path.relative_to(root).as_posix()
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
for line_number, line in enumerate(text.splitlines(), start=1):
|
||||
if DIRECT_BOT_API_RE.search(line):
|
||||
kind = surface_kind(relative_path)
|
||||
direct_calls.append(
|
||||
{
|
||||
"egress_surface_id": f"telegram_egress:{kind}:{relative_path}:{line_number}",
|
||||
"surface_kind": kind,
|
||||
"path": relative_path,
|
||||
"line": line_number,
|
||||
"line_hash": line_hash(relative_path, line_number, line),
|
||||
"sanitized_excerpt": sanitize_excerpt(line),
|
||||
"required_owner_fields": REQUIRED_OWNER_FIELDS,
|
||||
"reviewer_checks": REVIEWER_CHECKS,
|
||||
"outcome_lanes": OUTCOME_LANES,
|
||||
"blocked_actions": BLOCKED_ACTIONS,
|
||||
"owner_response_received": False,
|
||||
"owner_response_accepted": False,
|
||||
"formatter_convergence_accepted": False,
|
||||
"redaction_contract_accepted": False,
|
||||
"delivery_receipt_accepted": False,
|
||||
"direct_bot_api_migration_authorized": False,
|
||||
"telegram_send_authorized": False,
|
||||
"bot_api_call_authorized": False,
|
||||
"workflow_modification_authorized": False,
|
||||
"script_modification_authorized": False,
|
||||
"secret_value_collection_allowed": False,
|
||||
"raw_payload_storage_allowed": False,
|
||||
"production_write_authorized": False,
|
||||
"runtime_gate": False,
|
||||
"action_buttons_allowed": False,
|
||||
"not_authorization": True,
|
||||
}
|
||||
)
|
||||
text_lines = text.splitlines()
|
||||
for finding in scan_direct_bot_api_surfaces(relative_path, text):
|
||||
line_number = int(finding["line"])
|
||||
line = text_lines[line_number - 1] if line_number <= len(text_lines) else ""
|
||||
kind = surface_kind(relative_path)
|
||||
truth_classification = source_truth_classification(relative_path)
|
||||
direct_calls.append(
|
||||
{
|
||||
"egress_surface_id": f"telegram_egress:{kind}:{relative_path}:{line_number}",
|
||||
"surface_kind": kind,
|
||||
"source_truth_classification": truth_classification,
|
||||
"path": relative_path,
|
||||
"line": line_number,
|
||||
"line_hash": line_hash(relative_path, line_number, line),
|
||||
"method": finding["method"],
|
||||
"detection_kind": finding["detection_kind"],
|
||||
"function": finding["function"],
|
||||
"sanitized_excerpt": finding["sanitized_excerpt"],
|
||||
"required_owner_fields": REQUIRED_OWNER_FIELDS,
|
||||
"reviewer_checks": REVIEWER_CHECKS,
|
||||
"outcome_lanes": OUTCOME_LANES,
|
||||
"blocked_actions": BLOCKED_ACTIONS,
|
||||
"owner_response_received": False,
|
||||
"owner_response_accepted": False,
|
||||
"formatter_convergence_accepted": False,
|
||||
"redaction_contract_accepted": False,
|
||||
"delivery_receipt_accepted": False,
|
||||
"direct_bot_api_migration_authorized": False,
|
||||
"telegram_send_authorized": False,
|
||||
"bot_api_call_authorized": False,
|
||||
"workflow_modification_authorized": False,
|
||||
"script_modification_authorized": False,
|
||||
"secret_value_collection_allowed": False,
|
||||
"raw_payload_storage_allowed": False,
|
||||
"production_write_authorized": False,
|
||||
"runtime_execution_authorized": False,
|
||||
"workflow_execution_authorized": False,
|
||||
"runtime_gate": False,
|
||||
"action_buttons_allowed": False,
|
||||
"not_authorization": True,
|
||||
}
|
||||
)
|
||||
|
||||
for line_number, line in enumerate(text_lines, start=1):
|
||||
if GATEWAY_CALLSITE_RE.search(line):
|
||||
gateway_calls.append(
|
||||
{
|
||||
@@ -216,14 +745,56 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
}
|
||||
)
|
||||
|
||||
active_direct_calls = [
|
||||
item
|
||||
for item in direct_calls
|
||||
if item["source_truth_classification"] == "active_repo_source_truth"
|
||||
]
|
||||
frozen_legacy_direct_calls = [
|
||||
item
|
||||
for item in direct_calls
|
||||
if item["source_truth_classification"] == "frozen_legacy_source_truth"
|
||||
]
|
||||
direct_files = sorted({item["path"] for item in direct_calls})
|
||||
workflow_direct_calls = [item for item in direct_calls if item["surface_kind"] == "gitea_workflow_direct_bot_api"]
|
||||
ops_direct_calls = [item for item in direct_calls if item["surface_kind"] == "ops_script_direct_bot_api"]
|
||||
ci_direct_calls = [item for item in direct_calls if item["surface_kind"] == "ci_script_direct_bot_api"]
|
||||
api_direct_calls = [item for item in direct_calls if item["surface_kind"] == "api_direct_bot_api"]
|
||||
active_direct_files = sorted({item["path"] for item in active_direct_calls})
|
||||
frozen_legacy_direct_files = sorted(
|
||||
{item["path"] for item in frozen_legacy_direct_calls}
|
||||
)
|
||||
workflow_direct_calls = [
|
||||
item
|
||||
for item in direct_calls
|
||||
if item["surface_kind"] == "gitea_workflow_direct_bot_api"
|
||||
]
|
||||
ops_direct_calls = [
|
||||
item
|
||||
for item in direct_calls
|
||||
if item["surface_kind"] == "ops_script_direct_bot_api"
|
||||
]
|
||||
ci_direct_calls = [
|
||||
item
|
||||
for item in direct_calls
|
||||
if item["surface_kind"] == "ci_script_direct_bot_api"
|
||||
]
|
||||
api_direct_calls = [
|
||||
item for item in direct_calls if item["surface_kind"] == "api_direct_bot_api"
|
||||
]
|
||||
custom_direct_senders = [
|
||||
item
|
||||
for item in direct_calls
|
||||
if item["detection_kind"] == "custom_direct_sender"
|
||||
]
|
||||
direct_endpoints = [
|
||||
item
|
||||
for item in direct_calls
|
||||
if item["detection_kind"] == "direct_bot_api_endpoint"
|
||||
]
|
||||
telegram_gateway_path = root / "apps/api/src/services/telegram_gateway.py"
|
||||
telegram_gateway_text = telegram_gateway_path.read_text(encoding="utf-8", errors="replace")
|
||||
gateway_formatter_present = "normalize_telegram_send_message_payload" in telegram_gateway_text
|
||||
telegram_gateway_text = telegram_gateway_path.read_text(
|
||||
encoding="utf-8", errors="replace"
|
||||
)
|
||||
gateway_formatter_present = (
|
||||
"normalize_telegram_send_message_payload" in telegram_gateway_text
|
||||
)
|
||||
|
||||
return {
|
||||
"schema_version": "telegram_notification_egress_inventory_v1",
|
||||
@@ -236,12 +807,24 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
"scanned_file_count": len(files),
|
||||
"direct_bot_api_file_count": len(direct_files),
|
||||
"direct_bot_api_call_count": len(direct_calls),
|
||||
"active_direct_bot_api_file_count": len(active_direct_files),
|
||||
"active_direct_bot_api_call_count": len(active_direct_calls),
|
||||
"github_frozen_legacy_direct_bot_api_file_count": len(
|
||||
frozen_legacy_direct_files
|
||||
),
|
||||
"github_frozen_legacy_direct_bot_api_call_count": len(
|
||||
frozen_legacy_direct_calls
|
||||
),
|
||||
"direct_bot_api_endpoint_count": len(direct_endpoints),
|
||||
"custom_direct_sender_count": len(custom_direct_senders),
|
||||
"workflow_direct_bot_api_call_count": len(workflow_direct_calls),
|
||||
"ops_script_direct_bot_api_call_count": len(ops_direct_calls),
|
||||
"ci_script_direct_bot_api_call_count": len(ci_direct_calls),
|
||||
"api_direct_bot_api_call_count": len(api_direct_calls),
|
||||
"gateway_normalized_callsite_count": len(gateway_calls),
|
||||
"gateway_final_exit_formatter_present_count": 1 if gateway_formatter_present else 0,
|
||||
"gateway_final_exit_formatter_present_count": 1
|
||||
if gateway_formatter_present
|
||||
else 0,
|
||||
"required_owner_field_count": len(REQUIRED_OWNER_FIELDS),
|
||||
"reviewer_check_count": len(REVIEWER_CHECKS),
|
||||
"outcome_lane_count": len(OUTCOME_LANES),
|
||||
@@ -255,6 +838,7 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
"telegram_send_authorized_count": 0,
|
||||
"bot_api_call_authorized_count": 0,
|
||||
"workflow_modification_authorized_count": 0,
|
||||
"workflow_execution_authorized_count": 0,
|
||||
"script_modification_authorized_count": 0,
|
||||
"secret_value_collection_allowed_count": 0,
|
||||
"raw_payload_storage_allowed_count": 0,
|
||||
@@ -267,6 +851,8 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
"telegram_send_authorized": False,
|
||||
"bot_api_call_authorized": False,
|
||||
"workflow_modification_authorized": False,
|
||||
"workflow_execution_authorized": False,
|
||||
"github_workflow_execution_authorized": False,
|
||||
"script_modification_authorized": False,
|
||||
"secret_value_collection_allowed": False,
|
||||
"secret_hash_collection_allowed": False,
|
||||
@@ -282,7 +868,8 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
"direct_bot_api_calls": direct_calls,
|
||||
"gateway_normalized_callsite_refs": gateway_calls,
|
||||
"operator_interpretation": [
|
||||
"direct_bot_api_call_count 大於 0 代表仍有 workflow / ops / API 旁路可能繞過 TelegramGateway formatter。",
|
||||
"active_direct_bot_api_call_count 大於 0 代表仍有 active workflow / ops / API 旁路可能繞過 TelegramGateway formatter。",
|
||||
".github/workflows 僅列為 frozen_legacy_source_truth 供盤點，runtime_execution_authorized=false、workflow_execution_authorized=false，禁止執行。",
|
||||
"本清冊只建立 metadata-only egress surface，不送 Telegram、不修改 workflow / script、不讀 secret value。",
|
||||
"後續要收斂 direct Bot API 必須另走 owner response、formatter convergence、redaction contract、delivery receipt 與維護窗口。",
|
||||
],
|
||||
@@ -292,11 +879,15 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
def validate(root: Path) -> None:
|
||||
report = build_report(root)
|
||||
if report["summary"]["gateway_final_exit_formatter_present_count"] != 1:
|
||||
raise SystemExit("BLOCKED telegram egress inventory: gateway formatter not found")
|
||||
raise SystemExit(
|
||||
"BLOCKED telegram egress inventory: gateway formatter not found"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Build Telegram notification egress inventory")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build Telegram notification egress inventory"
|
||||
)
|
||||
parser.add_argument("--root", default=".", help="repository root")
|
||||
parser.add_argument("--output", help="write JSON snapshot")
|
||||
parser.add_argument("--generated-at", help="fixed generated_at timestamp")
|
||||
@@ -313,6 +904,9 @@ def main() -> None:
|
||||
print(
|
||||
"TELEGRAM_NOTIFICATION_EGRESS_INVENTORY_OK "
|
||||
f"direct_calls={report['summary']['direct_bot_api_call_count']} "
|
||||
f"active={report['summary']['active_direct_bot_api_call_count']} "
|
||||
f"frozen_legacy={report['summary']['github_frozen_legacy_direct_bot_api_call_count']} "
|
||||
f"custom={report['summary']['custom_direct_sender_count']} "
|
||||
f"files={report['summary']['direct_bot_api_file_count']} "
|
||||
f"workflow={report['summary']['workflow_direct_bot_api_call_count']} "
|
||||
f"ops={report['summary']['ops_script_direct_bot_api_call_count']} "
|
||||
|
||||
@@ -2,14 +2,16 @@
|
||||
"""檢查 Telegram 通知出口不可新增未登記 direct Bot API 旁路。
|
||||
|
||||
本 guard 只掃描 repo 原始碼與 committed snapshot，不讀 secret、不呼叫
|
||||
Telegram、不修改 workflow / script / API sender。既有 direct send 仍是待
|
||||
owner response 的基線；任何新增或變形的 direct Bot API endpoint 都必須先
|
||||
進 inventory / owner request / migration plan，而不是直接合併。
|
||||
Telegram、不修改或執行 workflow / script / API sender。Committed baseline
|
||||
僅供 drift 比對；目前 active source 的 direct endpoint 或 custom sender 必須
|
||||
真正為零。Local ``.github/workflows`` 只以 frozen legacy source truth 稽核，
|
||||
不得以 regex 漏掃、歷史基線或 frozen source 產生 false-green。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
@@ -22,14 +24,19 @@ from typing import Any
|
||||
|
||||
TAIPEI = timezone(timedelta(hours=8))
|
||||
|
||||
SOURCE_SNAPSHOT = Path("docs/security/telegram-notification-egress-inventory.snapshot.json")
|
||||
SOURCE_SNAPSHOT = Path(
|
||||
"docs/security/telegram-notification-egress-inventory.snapshot.json"
|
||||
)
|
||||
SCAN_ROOTS = (
|
||||
Path("agent99-control-plane.ps1"),
|
||||
Path(".gitea/workflows"),
|
||||
Path(".github/workflows"),
|
||||
Path("scripts/ops"),
|
||||
Path("scripts/ci"),
|
||||
Path("scripts/reboot-recovery"),
|
||||
Path("apps/api/src"),
|
||||
)
|
||||
SCAN_SUFFIXES = {".py", ".sh", ".js", ".yml", ".yaml"}
|
||||
SCAN_SUFFIXES = {".py", ".ps1", ".sh", ".js", ".yml", ".yaml"}
|
||||
GUARDED_BOT_METHODS = (
|
||||
"sendMessage",
|
||||
"sendDocument",
|
||||
@@ -48,6 +55,65 @@ BOT_ENDPOINT_RE = re.compile(
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
GUARDED_BOT_METHOD_RE = re.compile(
|
||||
r"\b(?P<method>"
|
||||
+ "|".join(re.escape(method) for method in GUARDED_BOT_METHODS)
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
COMPACT_BOT_ENDPOINT_RE = re.compile(
|
||||
r"api\.telegram\.org.{0,1024}?bot.{0,512}?/(?P<method>"
|
||||
+ "|".join(re.escape(method) for method in GUARDED_BOT_METHODS)
|
||||
+ r")",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DIRECT_HTTP_TRANSPORT_RE = re.compile(
|
||||
r"(?:"
|
||||
r"requests\s*\.\s*Session\s*\([^)]*\)\s*\.\s*(?:post|request|send)\s*\(|"
|
||||
r"requests\s*\.\s*(?:post|request)\s*\(|"
|
||||
r"httpx\s*\.\s*(?:AsyncClient|Client)\s*\([^)]*\)\s*\.\s*(?:post|request|send)\s*\(|"
|
||||
r"httpx\s*\.\s*(?:post|request)\s*\(|"
|
||||
r"(?:urllib\s*\.\s*request\s*\.\s*)?build_opener\s*\([^)]*\)\s*\.\s*open\s*\(|"
|
||||
r"urllib\s*\.\s*request\s*\.\s*(?:urlopen|Request)\s*\(|"
|
||||
r"\burlopen\s*\(|"
|
||||
r"\b(?:_?opener)\s*\.\s*open\s*\(|"
|
||||
r"\bInvoke-(?:WebRequest|RestMethod)\b|"
|
||||
r"\.\s*PostAsync\s*\(|"
|
||||
r"\.\s*Upload(?:String|Data|File)\s*\(|"
|
||||
r"\b(?:fetch|curl|wget)\b|"
|
||||
r"\b(?:_http_client|client|session)\s*\.\s*(?:post|request|send)\s*\("
|
||||
r")",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
TELEGRAM_CONTEXT_RE = re.compile(
|
||||
r"(?:telegram|bot[_-]?token|chat[_-]?id)", re.IGNORECASE
|
||||
)
|
||||
BOT_TOKEN_CONTEXT_RE = re.compile(
|
||||
r"(?:\bbot[_-]?token\b|\btelegram[_-]?(?:bot[_-]?)?token\b|\$Token\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
CHAT_CONTEXT_RE = re.compile(
|
||||
r"(?:\bchat[_-]?id\b|\$ChatId\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
POWERSHELL_FUNCTION_RE = re.compile(
|
||||
r"^function\s+(?P<name>[A-Za-z0-9_-]+)\s*\{", re.IGNORECASE | re.MULTILINE
|
||||
)
|
||||
CANONICAL_FINAL_EXITS = {
|
||||
(
|
||||
"apps/api/src/services/telegram_gateway.py",
|
||||
"TelegramGateway._send_request",
|
||||
),
|
||||
}
|
||||
_AST_IMPORT_NODE_TYPES = (ast.Import, ast.ImportFrom)
|
||||
_AST_FUNCTION_NODE_TYPES = (ast.FunctionDef, ast.AsyncFunctionDef)
|
||||
_AST_SEQUENCE_TARGET_NODE_TYPES = (ast.Tuple, ast.List)
|
||||
_AST_NON_MODULE_SCOPE_NODE_TYPES = (
|
||||
ast.ClassDef,
|
||||
ast.FunctionDef,
|
||||
ast.AsyncFunctionDef,
|
||||
ast.Lambda,
|
||||
)
|
||||
SECRET_INTERPOLATION_RE = re.compile(r"\$\{\{\s*secrets\.[^}]+\}\}")
|
||||
BOT_TOKEN_URL_RE = re.compile(
|
||||
r"api\.telegram\.org/bot.*?/(?P<method>"
|
||||
@@ -55,6 +121,12 @@ BOT_TOKEN_URL_RE = re.compile(
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
BOT_TOKEN_FRAGMENT_RE = re.compile(
|
||||
r"bot[^/\s\"']+/(?P<method>"
|
||||
+ "|".join(re.escape(method) for method in GUARDED_BOT_METHODS)
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def git_short_sha(root: Path) -> str:
|
||||
@@ -77,6 +149,10 @@ def iter_scannable_files(root: Path) -> list[Path]:
|
||||
absolute_root = root / scan_root
|
||||
if not absolute_root.exists():
|
||||
continue
|
||||
if absolute_root.is_file():
|
||||
if absolute_root.suffix in SCAN_SUFFIXES:
|
||||
files.append(absolute_root)
|
||||
continue
|
||||
for path in absolute_root.rglob("*"):
|
||||
if path.is_file() and path.suffix in SCAN_SUFFIXES:
|
||||
files.append(path)
|
||||
@@ -90,13 +166,436 @@ def sanitize_excerpt(line: str) -> str:
|
||||
lambda match: f"api.telegram.org/bot<redacted>/{match.group('method')}",
|
||||
excerpt,
|
||||
)
|
||||
excerpt = BOT_TOKEN_FRAGMENT_RE.sub(
|
||||
lambda match: f"bot<redacted>/{match.group('method')}",
|
||||
excerpt,
|
||||
)
|
||||
return excerpt[:180]
|
||||
|
||||
|
||||
def _compact_source(source: str) -> str:
|
||||
without_string_prefixes = re.sub(
|
||||
r"(?i)\b(?:fr|rf|f|r|u|b)(?=[\"'])",
|
||||
"",
|
||||
source,
|
||||
)
|
||||
return re.sub(r"[\s\"'`+()]+", "", without_string_prefixes)
|
||||
|
||||
|
||||
_PYTHON_HTTP_TRANSPORT_CALL_RE = re.compile(
|
||||
r"^(?:"
|
||||
r"requests\.(?:post|request)|"
|
||||
r"requests\.Session\(\)\.(?:post|request|send)|"
|
||||
r"httpx\.(?:post|request)|"
|
||||
r"httpx\.(?:AsyncClient|Client)\(\)\.(?:post|request|send)|"
|
||||
r"urllib\.request\.(?:urlopen|Request)|"
|
||||
r"urllib\.request\.build_opener\(\)\.open"
|
||||
r")$"
|
||||
)
|
||||
|
||||
|
||||
def _record_python_import_alias(
|
||||
node: ast.AST,
|
||||
aliases: dict[str, str],
|
||||
) -> None:
|
||||
if isinstance(node, ast.Import):
|
||||
for imported in node.names:
|
||||
bound_name = imported.asname or imported.name.split(".", 1)[0]
|
||||
aliases[bound_name] = (
|
||||
imported.name if imported.asname else bound_name
|
||||
)
|
||||
return
|
||||
|
||||
module = str(node.module or "").strip(".")
|
||||
if not module:
|
||||
return
|
||||
for imported in node.names:
|
||||
if imported.name == "*":
|
||||
continue
|
||||
aliases[imported.asname or imported.name] = (
|
||||
f"{module}.{imported.name}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_python_expression(
|
||||
node: ast.AST,
|
||||
aliases: dict[str, str],
|
||||
) -> str | None:
|
||||
if isinstance(node, ast.Name):
|
||||
return aliases.get(node.id, node.id)
|
||||
if isinstance(node, ast.Attribute):
|
||||
owner = _resolve_python_expression(node.value, aliases)
|
||||
return f"{owner}.{node.attr}" if owner else None
|
||||
if isinstance(node, ast.Call):
|
||||
callable_name = _resolve_python_expression(node.func, aliases)
|
||||
return f"{callable_name}()" if callable_name else None
|
||||
return None
|
||||
|
||||
|
||||
def _assignment_target_names(node: ast.AST) -> list[str]:
|
||||
if isinstance(node, ast.Name):
|
||||
return [node.id]
|
||||
if isinstance(node, _AST_SEQUENCE_TARGET_NODE_TYPES):
|
||||
return [
|
||||
name
|
||||
for child in node.elts
|
||||
for name in _assignment_target_names(child)
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def _python_parent_map(tree: ast.AST) -> dict[ast.AST, ast.AST]:
|
||||
return {
|
||||
child: parent
|
||||
for parent in ast.walk(tree)
|
||||
for child in ast.iter_child_nodes(parent)
|
||||
}
|
||||
|
||||
|
||||
def _python_function_qualified_name(
|
||||
function: ast.FunctionDef | ast.AsyncFunctionDef,
|
||||
parents: dict[ast.AST, ast.AST],
|
||||
) -> str:
|
||||
parts = [function.name]
|
||||
parent = parents.get(function)
|
||||
while parent is not None:
|
||||
if isinstance(parent, (ast.ClassDef, *_AST_FUNCTION_NODE_TYPES)):
|
||||
parts.append(parent.name)
|
||||
parent = parents.get(parent)
|
||||
return ".".join(reversed(parts))
|
||||
|
||||
|
||||
def _nearest_python_function(
|
||||
node: ast.AST,
|
||||
parents: dict[ast.AST, ast.AST],
|
||||
) -> ast.FunctionDef | ast.AsyncFunctionDef | None:
|
||||
parent = parents.get(node)
|
||||
while parent is not None:
|
||||
if isinstance(parent, _AST_FUNCTION_NODE_TYPES):
|
||||
return parent
|
||||
parent = parents.get(parent)
|
||||
return None
|
||||
|
||||
|
||||
def _is_python_module_scope(
|
||||
node: ast.AST,
|
||||
parents: dict[ast.AST, ast.AST],
|
||||
) -> bool:
|
||||
parent = parents.get(node)
|
||||
while parent is not None:
|
||||
if isinstance(parent, _AST_NON_MODULE_SCOPE_NODE_TYPES):
|
||||
return False
|
||||
parent = parents.get(parent)
|
||||
return True
|
||||
|
||||
|
||||
def _resolve_python_transport_calls(
|
||||
nodes: list[ast.AST],
|
||||
aliases: dict[str, str],
|
||||
) -> list[tuple[int, str]]:
|
||||
resolved_calls: list[tuple[int, str]] = []
|
||||
for node in sorted(
|
||||
nodes,
|
||||
key=lambda item: (
|
||||
int(getattr(item, "lineno", 0)),
|
||||
int(getattr(item, "col_offset", 0)),
|
||||
0 if isinstance(item, _AST_IMPORT_NODE_TYPES) else 1,
|
||||
),
|
||||
):
|
||||
if isinstance(node, _AST_IMPORT_NODE_TYPES):
|
||||
_record_python_import_alias(node, aliases)
|
||||
continue
|
||||
if isinstance(node, ast.Assign):
|
||||
resolved_value = _resolve_python_expression(node.value, aliases)
|
||||
for target in node.targets:
|
||||
for name in _assignment_target_names(target):
|
||||
if resolved_value:
|
||||
aliases[name] = resolved_value
|
||||
else:
|
||||
aliases.pop(name, None)
|
||||
continue
|
||||
if isinstance(node, ast.AnnAssign):
|
||||
resolved_value = (
|
||||
_resolve_python_expression(node.value, aliases)
|
||||
if node.value is not None
|
||||
else None
|
||||
)
|
||||
for name in _assignment_target_names(node.target):
|
||||
if resolved_value:
|
||||
aliases[name] = resolved_value
|
||||
else:
|
||||
aliases.pop(name, None)
|
||||
continue
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
qualified_call = _resolve_python_expression(node.func, aliases)
|
||||
if qualified_call and _PYTHON_HTTP_TRANSPORT_CALL_RE.fullmatch(
|
||||
qualified_call
|
||||
):
|
||||
resolved_calls.append((node.lineno, qualified_call))
|
||||
return sorted(set(resolved_calls))
|
||||
|
||||
|
||||
def _python_transport_calls_by_scope(
|
||||
text: str,
|
||||
) -> tuple[
|
||||
dict[tuple[str, int, int], list[tuple[int, str]]],
|
||||
list[tuple[int, str]],
|
||||
]:
|
||||
"""Resolve HTTP aliases in function and true module-level AST scopes."""
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return {}, []
|
||||
|
||||
parents = _python_parent_map(tree)
|
||||
module_aliases: dict[str, str] = {}
|
||||
module_nodes = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if _is_python_module_scope(node, parents)
|
||||
]
|
||||
module_calls = _resolve_python_transport_calls(
|
||||
module_nodes,
|
||||
module_aliases,
|
||||
)
|
||||
|
||||
transports: dict[tuple[str, int, int], list[tuple[int, str]]] = {}
|
||||
function_nodes = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, _AST_FUNCTION_NODE_TYPES)
|
||||
and node.end_lineno is not None
|
||||
]
|
||||
for function in function_nodes:
|
||||
aliases = dict(module_aliases)
|
||||
arguments = [
|
||||
*function.args.posonlyargs,
|
||||
*function.args.args,
|
||||
*function.args.kwonlyargs,
|
||||
]
|
||||
if function.args.vararg is not None:
|
||||
arguments.append(function.args.vararg)
|
||||
if function.args.kwarg is not None:
|
||||
arguments.append(function.args.kwarg)
|
||||
for argument in arguments:
|
||||
aliases.pop(argument.arg, None)
|
||||
|
||||
scoped_nodes = [
|
||||
node
|
||||
for node in ast.walk(function)
|
||||
if node is function
|
||||
or _nearest_python_function(node, parents) is function
|
||||
]
|
||||
resolved_calls = _resolve_python_transport_calls(scoped_nodes, aliases)
|
||||
|
||||
if resolved_calls:
|
||||
qualified_name = _python_function_qualified_name(function, parents)
|
||||
transports[
|
||||
(
|
||||
qualified_name,
|
||||
function.lineno,
|
||||
int(function.end_lineno),
|
||||
)
|
||||
] = resolved_calls
|
||||
return transports, module_calls
|
||||
|
||||
|
||||
def _transport_window_units(
|
||||
text: str,
|
||||
*,
|
||||
excluded_line_ranges: list[tuple[int, int]],
|
||||
) -> list[tuple[str, str, int, int, str]]:
|
||||
lines = text.splitlines()
|
||||
units: list[tuple[str, str, int, int, str]] = []
|
||||
for match in DIRECT_HTTP_TRANSPORT_RE.finditer(text):
|
||||
line_number = text.count("\n", 0, match.start()) + 1
|
||||
if any(start <= line_number <= end for start, end in excluded_line_ranges):
|
||||
continue
|
||||
start_line = max(1, line_number - 20)
|
||||
end_line = min(len(lines), line_number + 20)
|
||||
units.append(
|
||||
(
|
||||
"<module>",
|
||||
"<module>",
|
||||
start_line,
|
||||
end_line,
|
||||
"\n".join(lines[start_line - 1 : end_line]),
|
||||
)
|
||||
)
|
||||
return units
|
||||
|
||||
|
||||
def _source_units(
|
||||
relative_path: str,
|
||||
text: str,
|
||||
) -> list[tuple[str, str, int, int, str]]:
|
||||
lines = text.splitlines()
|
||||
units: list[tuple[str, str, int, int, str]] = []
|
||||
ranges: list[tuple[int, int]] = []
|
||||
if relative_path.endswith(".py"):
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
tree = None
|
||||
if tree is not None:
|
||||
parents = _python_parent_map(tree)
|
||||
nodes = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, _AST_FUNCTION_NODE_TYPES)
|
||||
and getattr(node, "end_lineno", None)
|
||||
]
|
||||
for node in sorted(
|
||||
nodes, key=lambda item: (item.lineno, item.end_lineno or item.lineno)
|
||||
):
|
||||
end_line = int(node.end_lineno or node.lineno)
|
||||
ranges.append((node.lineno, end_line))
|
||||
units.append(
|
||||
(
|
||||
node.name,
|
||||
_python_function_qualified_name(node, parents),
|
||||
node.lineno,
|
||||
end_line,
|
||||
"\n".join(lines[node.lineno - 1 : end_line]),
|
||||
)
|
||||
)
|
||||
elif relative_path.endswith(".ps1"):
|
||||
matches = list(POWERSHELL_FUNCTION_RE.finditer(text))
|
||||
for index, match in enumerate(matches):
|
||||
start_line = text.count("\n", 0, match.start()) + 1
|
||||
end_offset = (
|
||||
matches[index + 1].start() if index + 1 < len(matches) else len(text)
|
||||
)
|
||||
end_line = text.count("\n", 0, end_offset) + 1
|
||||
ranges.append((start_line, end_line))
|
||||
units.append(
|
||||
(
|
||||
match.group("name"),
|
||||
match.group("name"),
|
||||
start_line,
|
||||
end_line,
|
||||
text[match.start() : end_offset],
|
||||
)
|
||||
)
|
||||
return units + _transport_window_units(text, excluded_line_ranges=ranges)
|
||||
|
||||
|
||||
def scan_direct_bot_api_surfaces(
|
||||
relative_path: str,
|
||||
text: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Find direct Telegram transports, including split URLs and custom senders."""
|
||||
|
||||
source_lines = text.splitlines()
|
||||
findings: list[dict[str, Any]] = []
|
||||
seen: set[tuple[int, str, str]] = set()
|
||||
python_transports, module_transports = (
|
||||
_python_transport_calls_by_scope(text)
|
||||
if relative_path.endswith(".py")
|
||||
else ({}, [])
|
||||
)
|
||||
source_units = _source_units(relative_path, text)
|
||||
for line_number, _qualified_call in module_transports:
|
||||
start_line = max(1, line_number - 20)
|
||||
end_line = min(len(source_lines), line_number + 20)
|
||||
source_units.append(
|
||||
(
|
||||
"<module>",
|
||||
"<module>",
|
||||
start_line,
|
||||
end_line,
|
||||
"\n".join(source_lines[start_line - 1 : end_line]),
|
||||
)
|
||||
)
|
||||
|
||||
for (
|
||||
function_name,
|
||||
function_qualified,
|
||||
start_line,
|
||||
_end_line,
|
||||
source,
|
||||
) in source_units:
|
||||
if (relative_path, function_qualified) in CANONICAL_FINAL_EXITS:
|
||||
continue
|
||||
resolved_python_transports = (
|
||||
[
|
||||
item
|
||||
for item in module_transports
|
||||
if start_line <= item[0] <= _end_line
|
||||
]
|
||||
if function_qualified == "<module>"
|
||||
else python_transports.get(
|
||||
(function_qualified, start_line, _end_line),
|
||||
[],
|
||||
)
|
||||
)
|
||||
transport_match = DIRECT_HTTP_TRANSPORT_RE.search(source)
|
||||
if transport_match is None and not resolved_python_transports:
|
||||
continue
|
||||
method_match = GUARDED_BOT_METHOD_RE.search(source)
|
||||
endpoint_match = COMPACT_BOT_ENDPOINT_RE.search(_compact_source(source))
|
||||
telegram_named = bool(
|
||||
re.search(r"(?:telegram|bot)", function_name, re.IGNORECASE)
|
||||
)
|
||||
telegram_context = bool(TELEGRAM_CONTEXT_RE.search(source))
|
||||
token_and_chat_context = bool(
|
||||
BOT_TOKEN_CONTEXT_RE.search(source) and CHAT_CONTEXT_RE.search(source)
|
||||
)
|
||||
if (
|
||||
endpoint_match is None
|
||||
and method_match is None
|
||||
and not ((telegram_named and telegram_context) or token_and_chat_context)
|
||||
):
|
||||
continue
|
||||
|
||||
method = (
|
||||
endpoint_match.group("method")
|
||||
if endpoint_match is not None
|
||||
else method_match.group("method")
|
||||
if method_match is not None
|
||||
else "dynamic"
|
||||
)
|
||||
line_number = (
|
||||
resolved_python_transports[0][0]
|
||||
if resolved_python_transports
|
||||
else start_line + source[: transport_match.start()].count("\n")
|
||||
)
|
||||
detection_kind = (
|
||||
"direct_bot_api_endpoint"
|
||||
if endpoint_match is not None
|
||||
else "custom_direct_sender"
|
||||
)
|
||||
key = (line_number, method.lower(), detection_kind)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
source_line = (
|
||||
source_lines[line_number - 1] if line_number <= len(source_lines) else ""
|
||||
)
|
||||
findings.append(
|
||||
{
|
||||
"line": line_number,
|
||||
"method": method,
|
||||
"detection_kind": detection_kind,
|
||||
"function": function_name,
|
||||
"function_qualified": function_qualified,
|
||||
"sanitized_excerpt": sanitize_excerpt(source_line),
|
||||
}
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def signature(path: str, method: str, sanitized_excerpt: str) -> str:
|
||||
return f"{path}::{method.lower()}::{sanitized_excerpt}"
|
||||
|
||||
|
||||
def source_truth_classification(relative_path: str) -> str:
|
||||
if relative_path.startswith(".github/workflows/"):
|
||||
return "frozen_legacy_source_truth"
|
||||
return "active_repo_source_truth"
|
||||
|
||||
|
||||
def load_source_snapshot(root: Path) -> dict[str, Any]:
|
||||
snapshot_path = root / SOURCE_SNAPSHOT
|
||||
return json.loads(snapshot_path.read_text(encoding="utf-8"))
|
||||
@@ -105,9 +604,16 @@ def load_source_snapshot(root: Path) -> dict[str, Any]:
|
||||
def build_baseline(source_snapshot: dict[str, Any]) -> Counter[str]:
|
||||
baseline: Counter[str] = Counter()
|
||||
for item in source_snapshot.get("direct_bot_api_calls", []):
|
||||
if (
|
||||
source_truth_classification(str(item["path"]))
|
||||
== "frozen_legacy_source_truth"
|
||||
):
|
||||
continue
|
||||
excerpt = item.get("sanitized_excerpt", "")
|
||||
match = BOT_ENDPOINT_RE.search(excerpt)
|
||||
method = match.group("method") if match else "sendMessage"
|
||||
method = str(
|
||||
item.get("method") or (match.group("method") if match else "dynamic")
|
||||
)
|
||||
baseline[signature(item["path"], method, excerpt)] += 1
|
||||
return baseline
|
||||
|
||||
@@ -117,19 +623,25 @@ def scan_current_direct_endpoints(root: Path) -> list[dict[str, Any]]:
|
||||
for path in iter_scannable_files(root):
|
||||
relative_path = path.relative_to(root).as_posix()
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
for line_number, line in enumerate(text.splitlines(), start=1):
|
||||
for match in BOT_ENDPOINT_RE.finditer(line):
|
||||
method = match.group("method")
|
||||
sanitized = sanitize_excerpt(line)
|
||||
findings.append(
|
||||
{
|
||||
"path": relative_path,
|
||||
"line": line_number,
|
||||
"method": method,
|
||||
"sanitized_excerpt": sanitized,
|
||||
"signature": signature(relative_path, method, sanitized),
|
||||
}
|
||||
)
|
||||
for finding in scan_direct_bot_api_surfaces(relative_path, text):
|
||||
method = finding["method"]
|
||||
sanitized = finding["sanitized_excerpt"]
|
||||
findings.append(
|
||||
{
|
||||
"path": relative_path,
|
||||
"line": finding["line"],
|
||||
"method": method,
|
||||
"detection_kind": finding["detection_kind"],
|
||||
"function": finding["function"],
|
||||
"sanitized_excerpt": sanitized,
|
||||
"signature": signature(relative_path, method, sanitized),
|
||||
"source_truth_classification": source_truth_classification(
|
||||
relative_path
|
||||
),
|
||||
"runtime_execution_authorized": False,
|
||||
"workflow_execution_authorized": False,
|
||||
}
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
@@ -147,7 +659,17 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
generated = generated_at or datetime.now(TAIPEI).isoformat(timespec="seconds")
|
||||
source_snapshot = load_source_snapshot(root)
|
||||
baseline = build_baseline(source_snapshot)
|
||||
current_findings = scan_current_direct_endpoints(root)
|
||||
detected_findings = scan_current_direct_endpoints(root)
|
||||
current_findings = [
|
||||
item
|
||||
for item in detected_findings
|
||||
if item["source_truth_classification"] == "active_repo_source_truth"
|
||||
]
|
||||
frozen_legacy_findings = [
|
||||
item
|
||||
for item in detected_findings
|
||||
if item["source_truth_classification"] == "frozen_legacy_source_truth"
|
||||
]
|
||||
remaining_baseline = baseline.copy()
|
||||
new_bypass_findings: list[dict[str, Any]] = []
|
||||
|
||||
@@ -166,23 +688,48 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
current_files = sorted({item["path"] for item in current_findings})
|
||||
new_bypass_files = sorted({item["path"] for item in new_bypass_findings})
|
||||
counts_by_method = method_counts(current_findings)
|
||||
custom_direct_senders = [
|
||||
item
|
||||
for item in current_findings
|
||||
if item["detection_kind"] == "custom_direct_sender"
|
||||
]
|
||||
direct_endpoints = [
|
||||
item
|
||||
for item in current_findings
|
||||
if item["detection_kind"] == "direct_bot_api_endpoint"
|
||||
]
|
||||
source_summary = source_snapshot["summary"]
|
||||
|
||||
return {
|
||||
"schema_version": "telegram_notification_egress_no_new_bypass_guard_v1",
|
||||
"generated_at": generated,
|
||||
"git_commit": git_short_sha(root),
|
||||
"status": "pass_no_new_bypass" if not new_bypass_findings else "blocked_new_bypass_detected",
|
||||
"status": "pass_no_direct_or_custom_bypass"
|
||||
if not current_findings
|
||||
else "blocked_direct_or_custom_bypass_detected",
|
||||
"mode": "repo_source_scan_no_secret_value_no_telegram_send",
|
||||
"source_snapshot": SOURCE_SNAPSHOT.as_posix(),
|
||||
"guarded_roots": [path.as_posix() for path in SCAN_ROOTS],
|
||||
"guarded_bot_methods": list(GUARDED_BOT_METHODS),
|
||||
"summary": {
|
||||
"source_direct_bot_api_call_count": source_summary["direct_bot_api_call_count"],
|
||||
"source_direct_bot_api_file_count": source_summary["direct_bot_api_file_count"],
|
||||
"source_direct_bot_api_call_count": source_summary[
|
||||
"direct_bot_api_call_count"
|
||||
],
|
||||
"source_direct_bot_api_file_count": source_summary[
|
||||
"direct_bot_api_file_count"
|
||||
],
|
||||
"baseline_signature_count": sum(baseline.values()),
|
||||
"detected_direct_bot_api_call_count": len(detected_findings),
|
||||
"current_direct_bot_api_call_count": len(current_findings),
|
||||
"current_direct_bot_api_file_count": len(current_files),
|
||||
"current_direct_bot_api_endpoint_count": len(direct_endpoints),
|
||||
"current_custom_direct_sender_count": len(custom_direct_senders),
|
||||
"github_frozen_legacy_direct_bot_api_call_count": len(
|
||||
frozen_legacy_findings
|
||||
),
|
||||
"github_frozen_legacy_direct_bot_api_file_count": len(
|
||||
{item["path"] for item in frozen_legacy_findings}
|
||||
),
|
||||
"guarded_method_count": len(GUARDED_BOT_METHODS),
|
||||
"sendMessage_call_count": counts_by_method["sendMessage"],
|
||||
"sendDocument_call_count": counts_by_method["sendDocument"],
|
||||
@@ -203,7 +750,9 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
),
|
||||
"new_bypass_count": len(new_bypass_findings),
|
||||
"new_bypass_file_count": len(new_bypass_files),
|
||||
"removed_baseline_call_count": sum(item["removed_count"] for item in removed_baseline_signatures),
|
||||
"removed_baseline_call_count": sum(
|
||||
item["removed_count"] for item in removed_baseline_signatures
|
||||
),
|
||||
"runtime_gate_count": 0,
|
||||
"action_button_count": 0,
|
||||
},
|
||||
@@ -212,6 +761,8 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
"telegram_send_authorized": False,
|
||||
"bot_api_call_authorized": False,
|
||||
"workflow_modification_authorized": False,
|
||||
"workflow_execution_authorized": False,
|
||||
"github_workflow_execution_authorized": False,
|
||||
"script_modification_authorized": False,
|
||||
"api_sender_refactor_authorized": False,
|
||||
"secret_value_collection_allowed": False,
|
||||
@@ -225,15 +776,18 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
"not_authorization": True,
|
||||
},
|
||||
"current_direct_bot_api_calls": current_findings,
|
||||
"detected_direct_bot_api_calls": detected_findings,
|
||||
"frozen_legacy_direct_bot_api_calls": frozen_legacy_findings,
|
||||
"new_bypass_findings": new_bypass_findings,
|
||||
"removed_baseline_signatures": removed_baseline_signatures,
|
||||
"operator_interpretation": [
|
||||
"new_bypass_count 維持 0 才代表沒有新增未登記 Telegram Bot API 直送旁路。",
|
||||
"current_direct_bot_api_call_count 必須為 0，才代表沒有 Telegram direct/custom bypass。",
|
||||
".github/workflows 僅以 frozen_legacy_source_truth 盤點，不計入 active/new failure；runtime 與 workflow execution 均未授權且禁止執行。",
|
||||
(
|
||||
f"committed inventory 目前有 {source_summary['direct_bot_api_call_count']} 個 direct Bot API call site；"
|
||||
"若數值大於 0 仍是待 controlled migration 的基線，不代表已批准保留。"
|
||||
"baseline 只做 drift 比對，不能批准或隱藏目前 source 旁路。"
|
||||
),
|
||||
"sendDocument / sendPhoto / sendMediaGroup 等附件型出口若出現在 repo source，會被視為新增旁路並阻擋。",
|
||||
"分段 URL、requests/httpx/urllib、Invoke-WebRequest/Invoke-RestMethod 與自訂 sender 都會被掃描。",
|
||||
"本 guard 只讀 repo source 與 committed snapshot，不送 Telegram、不讀 Bot token、不修改 workflow / script / API sender。",
|
||||
],
|
||||
}
|
||||
@@ -242,10 +796,11 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]:
|
||||
def validate(root: Path) -> None:
|
||||
report = build_report(root)
|
||||
errors: list[str] = []
|
||||
if report["summary"]["new_bypass_count"]:
|
||||
for item in report["new_bypass_findings"]:
|
||||
if report["summary"]["current_direct_bot_api_call_count"]:
|
||||
for item in report["current_direct_bot_api_calls"]:
|
||||
errors.append(
|
||||
f"{item['path']}:{item['line']}: 新增未登記 Telegram Bot API 旁路 {item['method']}"
|
||||
f"{item['path']}:{item['line']}: Telegram direct/custom bypass "
|
||||
f"{item['detection_kind']} {item['method']} ({item['function']})"
|
||||
)
|
||||
|
||||
if errors:
|
||||
@@ -256,10 +811,14 @@ def validate(root: Path) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="檢查 Telegram 通知出口不可新增未登記 direct Bot API 旁路")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="檢查 Telegram 通知出口不可新增未登記 direct Bot API 旁路"
|
||||
)
|
||||
parser.add_argument("--root", default=".", help="repository root")
|
||||
parser.add_argument("--output", help="寫出 JSON 報告")
|
||||
parser.add_argument("--generated-at", help="固定 generated_at 時間，供 committed snapshot 使用")
|
||||
parser.add_argument(
|
||||
"--generated-at", help="固定 generated_at 時間，供 committed snapshot 使用"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(args.root).resolve()
|
||||
@@ -267,13 +826,18 @@ def main() -> None:
|
||||
if args.output:
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
validate(root)
|
||||
summary = report["summary"]
|
||||
print(
|
||||
"TELEGRAM_NOTIFICATION_EGRESS_NO_NEW_BYPASS_GUARD_OK "
|
||||
f"current={summary['current_direct_bot_api_call_count']} "
|
||||
f"custom={summary['current_custom_direct_sender_count']} "
|
||||
f"frozen_legacy={summary['github_frozen_legacy_direct_bot_api_call_count']} "
|
||||
f"baseline={summary['baseline_signature_count']} "
|
||||
f"new={summary['new_bypass_count']} "
|
||||
f"sendDocument={summary['sendDocument_call_count']} "
|
||||
|
||||
Reference in New Issue
Block a user