perf(runtime): isolate workers and compress web assets

services/codebase_modularization_performance_service.py

@@ -0,0 +1,217 @@
+"""Automated codebase modularization and frontend payload inventory."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Iterable
+
+
+ROOT = Path(__file__).resolve().parents[1]
+TAIPEI_TZ = timezone(timedelta(hours=8))
+PYTHON_DEBT_LINES = 800
+FRONTEND_P0_BYTES = 50_000
+FRONTEND_P1_BYTES = 20_000
+IN_PROGRESS_PYTHON_PATHS = {
+    "app.py",
+    "routes/openclaw_bot_routes.py",
+    "run_scheduler.py",
+}
+IGNORED_PARTS = {
+    ".git",
+    ".pytest_cache",
+    ".venv",
+    "__pycache__",
+    "backups",
+    "tests",
+    "venv",
+}
+
+
+def _iter_python_files(root: Path) -> Iterable[Path]:
+    for path in root.rglob("*.py"):
+        relative = path.relative_to(root)
+        if any(part in IGNORED_PARTS for part in relative.parts):
+            continue
+        if relative.parts[:2] in {("docs", "design"), ("docs", "design_audit_frontend")}:
+            continue
+        yield path
+
+
+def _iter_frontend_files(root: Path) -> Iterable[Path]:
+    for folder in ("templates", "static", "web/templates", "web/static"):
+        base = root / folder
+        if not base.exists():
+            continue
+        for path in base.rglob("*"):
+            if path.is_file() and path.suffix.lower() in {".html", ".css", ".js"}:
+                yield path
+
+
+def _line_count(path: Path) -> int:
+    with path.open(encoding="utf-8", errors="ignore") as handle:
+        return sum(1 for _ in handle)
+
+
+def _priority_for_python(lines: int) -> str:
+    if lines >= 2_000:
+        return "P0"
+    if lines >= PYTHON_DEBT_LINES:
+        return "P1"
+    return "P2"
+
+
+def _priority_for_frontend(size_bytes: int) -> str:
+    return "P0" if size_bytes >= FRONTEND_P0_BYTES else "P1"
+
+
+def _split_direction(path: str) -> str:
+    if path.startswith("routes/"):
+        return "separate route glue, queries, commands, and projections"
+    if path == "scheduler.py" or path == "run_scheduler.py":
+        return "separate task registry, domain jobs, runtime workers, and receipts"
+    if path.startswith("services/"):
+        return "separate policy, adapters, execution, verification, and reporting"
+    if path == "app.py":
+        return "extract app factory, extensions, blueprint registry, and startup guards"
+    return "separate cohesive domain responsibilities behind stable interfaces"
+
+
+def _work_item_id(kind: str, path: str) -> str:
+    digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:10]
+    return f"{kind}-{digest}"
+
+
+def _runtime_controls(root: Path) -> list[dict]:
+    app_source = (root / "app.py").read_text(encoding="utf-8")
+    bot_source = (root / "routes/openclaw_bot_routes.py").read_text(encoding="utf-8")
+    learning_source = (root / "services/openclaw_learning_service.py").read_text(encoding="utf-8")
+    login_source = (root / "templates/login.html").read_text(encoding="utf-8")
+    registration = bot_source[bot_source.index("@openclaw_bot_bp.record_once") :]
+    return [
+        {
+            "id": "web_no_eager_scheduler_import",
+            "passed": "from scheduler import" not in app_source,
+        },
+        {
+            "id": "openclaw_web_registration_no_scheduler_boot",
+            "passed": "start_scheduler()" not in registration,
+        },
+        {
+            "id": "embedding_worker_explicit_runtime_owner",
+            "passed": "Thread(target=_embedding_worker_loop" not in learning_source,
+        },
+        {
+            "id": "text_compression_enabled",
+            "passed": "Compress(app)" in app_source and "COMPRESS_ALGORITHM" in app_source,
+        },
+        {
+            "id": "versioned_static_immutable",
+            "passed": "max-age=31536000, immutable" in app_source,
+        },
+        {
+            "id": "login_has_no_third_party_assets",
+            "passed": "https://" not in login_source and "<style>" not in login_source,
+        },
+    ]
+
+
+def build_codebase_inventory(*, root: Path | None = None, now: datetime | None = None) -> dict:
+    project_root = (root or ROOT).resolve()
+    python_modules = []
+    total_python_lines = 0
+    debt_lines = 0
+    for path in _iter_python_files(project_root):
+        relative = path.relative_to(project_root).as_posix()
+        lines = _line_count(path)
+        total_python_lines += lines
+        debt_lines += max(0, lines - (PYTHON_DEBT_LINES - 1))
+        python_modules.append({"path": relative, "lines": lines})
+
+    large_modules = [
+        {
+            **module,
+            "priority": _priority_for_python(module["lines"]),
+            "work_item_id": _work_item_id("MOD", module["path"]),
+            "status": (
+                "in_progress" if module["path"] in IN_PROGRESS_PYTHON_PATHS else "not_started"
+            ),
+            "split_direction": _split_direction(module["path"]),
+        }
+        for module in python_modules
+        if module["lines"] >= PYTHON_DEBT_LINES
+    ]
+    large_modules.sort(key=lambda item: (-item["lines"], item["path"]))
+
+    frontend_assets = []
+    total_frontend_bytes = 0
+    for path in _iter_frontend_files(project_root):
+        relative = path.relative_to(project_root).as_posix()
+        size_bytes = path.stat().st_size
+        total_frontend_bytes += size_bytes
+        if size_bytes >= FRONTEND_P1_BYTES:
+            frontend_assets.append({
+                "path": relative,
+                "size_bytes": size_bytes,
+                "priority": _priority_for_frontend(size_bytes),
+                "work_item_id": _work_item_id("WEB", relative),
+                "status": "not_started",
+                "split_direction": "extract reusable components and defer non-critical payloads",
+            })
+    frontend_assets.sort(key=lambda item: (-item["size_bytes"], item["path"]))
+
+    controls = _runtime_controls(project_root)
+    passed_controls = sum(1 for item in controls if item["passed"])
+    line_health = 1.0 - (debt_lines / max(1, total_python_lines))
+    control_health = passed_controls / max(1, len(controls))
+    completion = round(max(0.0, min(1.0, line_health * 0.7 + control_health * 0.3)) * 100, 1)
+    p0_count = sum(1 for item in large_modules + frontend_assets if item["priority"] == "P0")
+    all_work_items = large_modules + frontend_assets
+    work_item_status_counts = {
+        status: sum(1 for item in all_work_items if item["status"] == status)
+        for status in ("completed", "in_progress", "not_started")
+    }
+    generated_at = (now or datetime.now(TAIPEI_TZ)).astimezone(TAIPEI_TZ).isoformat()
+
+    return {
+        "schema": "ewoooc_codebase_modularization_performance_v1",
+        "generated_at": generated_at,
+        "source_root": str(project_root),
+        "summary": {
+            "production_python_module_count": len(python_modules),
+            "production_python_lines": total_python_lines,
+            "python_debt_lines_over_799": debt_lines,
+            "large_python_module_count": len(large_modules),
+            "frontend_asset_bytes": total_frontend_bytes,
+            "large_frontend_asset_count": len(frontend_assets),
+            "runtime_control_passed": passed_controls,
+            "runtime_control_total": len(controls),
+            "p0_work_item_count": p0_count,
+            "work_item_status_counts": work_item_status_counts,
+            "program_completion_percent": completion,
+            "runtime_closure": "closed" if p0_count == 0 and passed_controls == len(controls) else "partial",
+        },
+        "runtime_controls": controls,
+        "python_work_items": large_modules,
+        "frontend_work_items": frontend_assets,
+    }
+
+
+def write_codebase_inventory_receipt(result: dict, *, path: Path | None = None) -> Path:
+    target = path or Path(
+        os.getenv(
+            "CODEBASE_INVENTORY_RECEIPT_PATH",
+            ROOT / "data/ai_automation/codebase_inventory/latest.json",
+        )
+    )
+    target.parent.mkdir(parents=True, exist_ok=True)
+    temporary = target.with_suffix(target.suffix + ".tmp")
+    temporary.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    temporary.replace(target)
+    return target
+
+
+__all__ = ["build_codebase_inventory", "write_codebase_inventory_receipt"]