231 lines
8.1 KiB
Python
231 lines
8.1 KiB
Python
"""Automated codebase modularization and frontend payload inventory."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
TAIPEI_TZ = timezone(timedelta(hours=8))
|
|
PYTHON_DEBT_LINES = 800
|
|
FRONTEND_P0_BYTES = 50_000
|
|
FRONTEND_P1_BYTES = 20_000
|
|
IN_PROGRESS_PYTHON_PATHS = {
|
|
"app.py",
|
|
"routes/openclaw_bot_routes.py",
|
|
"run_scheduler.py",
|
|
}
|
|
IGNORED_PARTS = {
|
|
".claude",
|
|
".docker",
|
|
".git",
|
|
".gitea",
|
|
".github",
|
|
".pytest_cache",
|
|
".venv",
|
|
"__pycache__",
|
|
"aiops-core",
|
|
"backups",
|
|
"data",
|
|
"docs",
|
|
"k8s",
|
|
"logs",
|
|
"memory",
|
|
"n8n-workflows",
|
|
"node_modules",
|
|
"runtime_artifacts",
|
|
"secrets",
|
|
"ssl",
|
|
"tests",
|
|
"venv",
|
|
}
|
|
|
|
|
|
def _iter_python_files(root: Path) -> Iterable[Path]:
|
|
for path in root.rglob("*.py"):
|
|
relative = path.relative_to(root)
|
|
if any(part in IGNORED_PARTS for part in relative.parts):
|
|
continue
|
|
yield path
|
|
|
|
|
|
def _iter_frontend_files(root: Path) -> Iterable[Path]:
|
|
for folder in ("templates", "static", "web/templates", "web/static"):
|
|
base = root / folder
|
|
if not base.exists():
|
|
continue
|
|
for path in base.rglob("*"):
|
|
if path.is_file() and path.suffix.lower() in {".html", ".css", ".js"}:
|
|
yield path
|
|
|
|
|
|
def _line_count(path: Path) -> int:
|
|
with path.open(encoding="utf-8", errors="ignore") as handle:
|
|
return sum(1 for _ in handle)
|
|
|
|
|
|
def _priority_for_python(lines: int) -> str:
|
|
if lines >= 2_000:
|
|
return "P0"
|
|
if lines >= PYTHON_DEBT_LINES:
|
|
return "P1"
|
|
return "P2"
|
|
|
|
|
|
def _priority_for_frontend(size_bytes: int) -> str:
|
|
return "P0" if size_bytes >= FRONTEND_P0_BYTES else "P1"
|
|
|
|
|
|
def _split_direction(path: str) -> str:
|
|
if path.startswith("routes/"):
|
|
return "separate route glue, queries, commands, and projections"
|
|
if path == "scheduler.py" or path == "run_scheduler.py":
|
|
return "separate task registry, domain jobs, runtime workers, and receipts"
|
|
if path.startswith("services/"):
|
|
return "separate policy, adapters, execution, verification, and reporting"
|
|
if path == "app.py":
|
|
return "extract app factory, extensions, blueprint registry, and startup guards"
|
|
return "separate cohesive domain responsibilities behind stable interfaces"
|
|
|
|
|
|
def _work_item_id(kind: str, path: str) -> str:
|
|
digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:10]
|
|
return f"{kind}-{digest}"
|
|
|
|
|
|
def _runtime_controls(root: Path) -> list[dict]:
|
|
app_source = (root / "app.py").read_text(encoding="utf-8")
|
|
bot_source = (root / "routes/openclaw_bot_routes.py").read_text(encoding="utf-8")
|
|
learning_source = (root / "services/openclaw_learning_service.py").read_text(encoding="utf-8")
|
|
login_source = (root / "templates/login.html").read_text(encoding="utf-8")
|
|
registration = bot_source[bot_source.index("@openclaw_bot_bp.record_once") :]
|
|
return [
|
|
{
|
|
"id": "web_no_eager_scheduler_import",
|
|
"passed": "from scheduler import" not in app_source,
|
|
},
|
|
{
|
|
"id": "openclaw_web_registration_no_scheduler_boot",
|
|
"passed": "start_scheduler()" not in registration,
|
|
},
|
|
{
|
|
"id": "embedding_worker_explicit_runtime_owner",
|
|
"passed": "Thread(target=_embedding_worker_loop" not in learning_source,
|
|
},
|
|
{
|
|
"id": "text_compression_enabled",
|
|
"passed": "Compress(app)" in app_source and "COMPRESS_ALGORITHM" in app_source,
|
|
},
|
|
{
|
|
"id": "versioned_static_immutable",
|
|
"passed": "max-age=31536000, immutable" in app_source,
|
|
},
|
|
{
|
|
"id": "login_has_no_third_party_assets",
|
|
"passed": "https://" not in login_source and "<style>" not in login_source,
|
|
},
|
|
]
|
|
|
|
|
|
def build_codebase_inventory(*, root: Path | None = None, now: datetime | None = None) -> dict:
|
|
project_root = (root or ROOT).resolve()
|
|
python_modules = []
|
|
total_python_lines = 0
|
|
debt_lines = 0
|
|
for path in _iter_python_files(project_root):
|
|
relative = path.relative_to(project_root).as_posix()
|
|
lines = _line_count(path)
|
|
total_python_lines += lines
|
|
debt_lines += max(0, lines - (PYTHON_DEBT_LINES - 1))
|
|
python_modules.append({"path": relative, "lines": lines})
|
|
|
|
large_modules = [
|
|
{
|
|
**module,
|
|
"priority": _priority_for_python(module["lines"]),
|
|
"work_item_id": _work_item_id("MOD", module["path"]),
|
|
"status": (
|
|
"in_progress" if module["path"] in IN_PROGRESS_PYTHON_PATHS else "not_started"
|
|
),
|
|
"split_direction": _split_direction(module["path"]),
|
|
}
|
|
for module in python_modules
|
|
if module["lines"] >= PYTHON_DEBT_LINES
|
|
]
|
|
large_modules.sort(key=lambda item: (-item["lines"], item["path"]))
|
|
|
|
frontend_assets = []
|
|
total_frontend_bytes = 0
|
|
for path in _iter_frontend_files(project_root):
|
|
relative = path.relative_to(project_root).as_posix()
|
|
size_bytes = path.stat().st_size
|
|
total_frontend_bytes += size_bytes
|
|
if size_bytes >= FRONTEND_P1_BYTES:
|
|
frontend_assets.append({
|
|
"path": relative,
|
|
"size_bytes": size_bytes,
|
|
"priority": _priority_for_frontend(size_bytes),
|
|
"work_item_id": _work_item_id("WEB", relative),
|
|
"status": "not_started",
|
|
"split_direction": "extract reusable components and defer non-critical payloads",
|
|
})
|
|
frontend_assets.sort(key=lambda item: (-item["size_bytes"], item["path"]))
|
|
|
|
controls = _runtime_controls(project_root)
|
|
passed_controls = sum(1 for item in controls if item["passed"])
|
|
line_health = 1.0 - (debt_lines / max(1, total_python_lines))
|
|
control_health = passed_controls / max(1, len(controls))
|
|
completion = round(max(0.0, min(1.0, line_health * 0.7 + control_health * 0.3)) * 100, 1)
|
|
p0_count = sum(1 for item in large_modules + frontend_assets if item["priority"] == "P0")
|
|
all_work_items = large_modules + frontend_assets
|
|
work_item_status_counts = {
|
|
status: sum(1 for item in all_work_items if item["status"] == status)
|
|
for status in ("completed", "in_progress", "not_started")
|
|
}
|
|
generated_at = (now or datetime.now(TAIPEI_TZ)).astimezone(TAIPEI_TZ).isoformat()
|
|
|
|
return {
|
|
"schema": "ewoooc_codebase_modularization_performance_v1",
|
|
"generated_at": generated_at,
|
|
"source_root": str(project_root),
|
|
"summary": {
|
|
"production_python_module_count": len(python_modules),
|
|
"production_python_lines": total_python_lines,
|
|
"python_debt_lines_over_799": debt_lines,
|
|
"large_python_module_count": len(large_modules),
|
|
"frontend_asset_bytes": total_frontend_bytes,
|
|
"large_frontend_asset_count": len(frontend_assets),
|
|
"runtime_control_passed": passed_controls,
|
|
"runtime_control_total": len(controls),
|
|
"p0_work_item_count": p0_count,
|
|
"work_item_status_counts": work_item_status_counts,
|
|
"program_completion_percent": completion,
|
|
"runtime_closure": "closed" if p0_count == 0 and passed_controls == len(controls) else "partial",
|
|
},
|
|
"runtime_controls": controls,
|
|
"python_work_items": large_modules,
|
|
"frontend_work_items": frontend_assets,
|
|
}
|
|
|
|
|
|
def write_codebase_inventory_receipt(result: dict, *, path: Path | None = None) -> Path:
|
|
target = path or Path(
|
|
os.getenv(
|
|
"CODEBASE_INVENTORY_RECEIPT_PATH",
|
|
ROOT / "data/ai_automation/codebase_inventory/latest.json",
|
|
)
|
|
)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = target.with_suffix(target.suffix + ".tmp")
|
|
temporary.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(target)
|
|
return target
|
|
|
|
|
|
__all__ = ["build_codebase_inventory", "write_codebase_inventory_receipt"]
|