Merge pull request #171 from devdanzin/fleet-observability-stats-agent

feat(fleet): per-session stats sidecar (StatsAgent) + `fleet report` observability

Author: devdanzin

fleet/README.md

@@ -34,6 +34,8 @@ and capture their logs — no VMs or containers needed.
 ```bash
 sudo ./fleet up            # start (nproc-1) instances; or: sudo ./fleet up 8
 ./fleet status             # per-instance state + crashes kept + NEW candidates
+./fleet report             # rich observability: sessions, throughput, crash taxonomy, disk, health
+./fleet report 3           # ...for one instance;  add --watch for a live view, --json for scripts
 ./fleet finds              # list the oomNEW dirs across the whole fleet
 ./fleet tail 3             # follow instance 3's output live
 sudo ./fleet down          # stop everything

fleet/fleet

@@ -6,6 +6,7 @@
 #   sudo ./fleet down        stop + disable all instances
 #   ./fleet status          per-instance state + crash/NEW-find counts
 #   ./fleet finds           list this fleet's oomNEW (new-bug candidate) dirs
+#   ./fleet report [N]      observability report: per-instance + campaign (--watch, --json, --no-systemd)
 #   ./fleet triage          dedupe oomNEW/oomSEGV candidates across instances (needs INGEST); extra args pass to ingest (e.g. --gdb)
 #   ./fleet tail [N]        follow instance N's log (default: 1)
 #   sudo ./fleet restart    restart all instances (e.g. to pick up a new catalog)
@@ -151,6 +152,15 @@ cmd_triage(){
     fi
 }
 
+cmd_report(){
+    # Observability report (per-instance + campaign). Pure-Python reader; put the repo on
+    # PYTHONPATH so RUNNER_PY can import fusil.python.fleet_report even when fusil isn't
+    # pip-installed in that venv. All args (N, --watch, --json, --no-systemd) pass through.
+    local repo; repo="$(dirname "$(dirname "$FUSIL_PY")")"
+    PYTHONPATH="$repo${PYTHONPATH:+:$PYTHONPATH}" \
+        "$RUNNER_PY" -m fusil.python.fleet_report --fleet-dir "$FLEET_DIR" "$@"
+}
+
 cmd_tail(){
     local i="${1:-1}" dir; dir="$(inst_dir "$i")"
     if [ "${LOG:-file}" = "none" ]; then
@@ -170,6 +180,7 @@ case "${1:-status}" in
     status)   cmd_status ;;
     finds)    cmd_finds ;;
     triage)   shift; cmd_triage "$@" ;;
+    report)   shift; cmd_report "$@" ;;
     tail)     shift; cmd_tail "$@" ;;
-    *)        sed -n '2,13p' "$HERE/fleet" | sed 's/^# \{0,1\}//' ;;
+    *)        sed -n '2,14p' "$HERE/fleet" | sed 's/^# \{0,1\}//' ;;
 esac

fusil/python/__init__.py

@@ -406,6 +406,12 @@ def setupProject(self) -> None:
         self.source = PythonSource(
             project, self.options, source_output_path=self.options.source_output_path
         )
+        # Score-neutral observer: fold each finished session into the run dir's
+        # fusil_stats.json sidecar (read back by `fleet report`). Parent-side only; does not
+        # override getScore(), so it never affects scoring or the generated source.
+        from fusil.python.stats_agent import StatsAgent
+
+        StatsAgent(project, self.source)
         process = PythonProcess(
             project,
             self.options,

fusil/python/_report_format.py

@@ -0,0 +1,62 @@
+"""Small, dependency-free text-formatting helpers for the fleet reporter.
+
+Adapted from lafleur's observability layer (``lafleur/utils.py``, ``lafleur/report.py``) --
+copied rather than imported so fusil takes no dependency on lafleur. Pure stdlib.
+"""
+
+from __future__ import annotations
+
+import os
+
+
+def human_bytes(n: float | None) -> str:
+    """1536 -> '1.5K', 1234567890 -> '1.1G'. None/negative -> '-'."""
+    if n is None or n < 0:
+        return "-"
+    n = float(n)
+    for unit in ("B", "K", "M", "G", "T", "P"):
+        if n < 1024 or unit == "P":
+            if unit == "B":
+                return "%dB" % int(n)
+            return "%.1f%s" % (n, unit)
+        n /= 1024
+    return "%.1fP" % n
+
+
+def format_duration(seconds: float | None) -> str:
+    """90061 -> '1d 1h', 305 -> '5m 5s', None -> '-'. Two most-significant units."""
+    if seconds is None or seconds < 0:
+        return "-"
+    seconds = int(seconds)
+    days, rem = divmod(seconds, 86400)
+    hours, rem = divmod(rem, 3600)
+    minutes, secs = divmod(rem, 60)
+    parts = []
+    if days:
+        parts += ["%dd" % days, "%dh" % hours]
+    elif hours:
+        parts += ["%dh" % hours, "%dm" % minutes]
+    elif minutes:
+        parts += ["%dm" % minutes, "%ds" % secs]
+    else:
+        parts = ["%ds" % secs]
+    return " ".join(parts[:2])
+
+
+def format_rate(count: int, seconds: float | None) -> str:
+    """Per-minute rate, e.g. 3092 sessions over 2h -> '25.8/m'. '-' if no elapsed time."""
+    if not seconds or seconds <= 0:
+        return "-"
+    return "%.1f/m" % (count / (seconds / 60.0))
+
+
+def dir_size_bytes(path: str) -> int:
+    """Total size of a directory tree in bytes (best-effort; unreadable entries skipped)."""
+    total = 0
+    for root, _dirs, files in os.walk(path, onerror=lambda _e: None):
+        for name in files:
+            try:
+                total += os.lstat(os.path.join(root, name)).st_size
+            except OSError:
+                pass
+    return total