refactor(compiler): workspace cleanup, PDK ergonomics, and FFI hardening

Cargo.toml

@@ -11,6 +11,19 @@ edition = "2024"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/dylan-sutton-chavez/edge-python/"
 
+# Single source of truth for third-party versions. Members opt in with
+# `dep = { workspace = true }` so a bump touches one line workspace-wide
+# and the lockfile cannot grow accidental version splits.
+[workspace.dependencies]
+hashbrown = { version = "0.17", default-features = false }
+itoa = "1"
+lol_alloc = "0.4"
+proc-macro2 = "1"
+quote = "1"
+syn = { version = "2", features = ["full"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
 [profile.release]
 opt-level = "z"
 lto = true

add-bell-backspace-form-vtab-escapes.txt

@@ -0,0 +1,30 @@
+Branch: add-bell-backspace-form-vtab-escapes
+Tech debt: D70 — missing \a \b \f \v escapes in unescape and parse_bytes_literal.
+
+================================================================================
+Commit 1: add bell backspace form-feed vertical-tab escapes
+================================================================================
+Files:
+  - compiler/src/modules/parser/types.rs
+  - documentation/implementation/lexical.md
+
+What was wrong:
+  unescape (parser/types.rs:422+) had cases for \n \t \r \\ \' \" \x \u \U
+  and 1-3 digit octal, but no cases for \a \b \f \v. These are valid Python
+  escape sequences for ASCII control characters BEL, BS, FF, VT. Without a
+  matching arm they fell through to the "Some(c) => out.push('\\'); out.push(c);"
+  branch, so "\\a" stayed as the two characters '\' and 'a'.
+
+  parse_bytes_literal had the same gap.
+
+Fix:
+  Added \a -> 0x07, \b -> 0x08, \f -> 0x0C, \v -> 0x0B in both unescape (str)
+  and parse_bytes_literal (bytes), keeping the existing match-arm style.
+
+Documentation:
+  documentation/implementation/lexical.md line 78 explicitly enumerates the
+  recognised escapes; updating it was strictly necessary since omitting the
+  new escapes would leave the spec incomplete. Added \a \b \f \v to the
+  recognised-escapes list in the same compact inline format.
+
+Tests: cargo test -> 6/6 passed.

add-edge-pdk-prelude.txt

@@ -0,0 +1,52 @@
+Branch: add-edge-pdk-prelude
+Tech debt: D56 — `use edge_pdk::*;` glob-imports #[doc(hidden)] symbols.
+
+================================================================================
+Commit 1: introduce edge_pdk::prelude
+================================================================================
+File: edge-pdk/src/lib.rs
+
+What was wrong:
+  Plugin authors are expected to write `use edge_pdk::*;` (this is the
+  pattern in the wasm-abi doc and the slugify-mod example). The glob
+  brings in everything that's `pub` at the crate root, including the
+  two macro-internal symbols documented as #[doc(hidden)]:
+
+    pub fn __stash_error(e: Error)        // line 74
+    pub extern "C" fn __edge_alloc(...)   // line 111
+
+  These symbols exist because `#[plugin_fn]` expansion calls them; they
+  are not part of the user-facing API. But #[doc(hidden)] only hides
+  them from rustdoc — it does not exclude them from a glob import. So
+  every plugin author silently gets `__stash_error` and `__edge_alloc`
+  in their namespace, and any future rename of those internals is a
+  breaking change to plugins that don't use them.
+
+Fix:
+  Added a curated prelude module:
+
+    pub mod prelude {
+        pub use crate::{plugin_fn, Handle, Value, Error, Result,
+                        FromValue, IntoValue};
+    }
+
+  This is the standard Rust prelude pattern (cf. std::prelude). Plugin
+  authors can switch to `use edge_pdk::prelude::*;` to opt out of the
+  glob's noise. The legacy `use edge_pdk::*;` path is unchanged and
+  still works for existing plugins; the prelude is purely additive.
+
+  A /* ... */ doc comment in the lexer/parser style explains the
+  intent so contributors know where to extend the surface.
+
+  This addresses just the prelude half of D56. The companion fix —
+  moving __stash_error / __edge_alloc into a `__internals` submodule
+  so they cannot leak through any glob — is a breaking change for the
+  proc-macro and is left for a separate branch.
+
+Documentation:
+  reference/wasm-abi.md examples use `use edge_pdk::*;` which continues
+  to work unchanged. Recommending the prelude in those examples is a
+  pure migration suggestion, not a correctness fix; skipped to keep
+  the change strictly additive.
+
+Build & tests: cargo build -p edge-pdk -> ok; cargo test -> 6/6 passed.

add-pdk-module-bootstrap-macro.txt

@@ -0,0 +1,58 @@
+Branch: add-pdk-module-bootstrap-macro
+Tech debt: D55 — every plugin author re-declares the same allocator + panic_handler boilerplate.
+
+================================================================================
+Commit 1: add edge_pdk::module! bootstrap macro
+================================================================================
+Files:
+  - edge-pdk/Cargo.toml (wasm32 lol_alloc dep)
+  - edge-pdk/src/lib.rs (re-export + macro_rules! module)
+
+What was wrong:
+  Each plugin's lib.rs duplicated four lines of boilerplate:
+
+    extern crate alloc;
+    #[global_allocator]
+    static A: lol_alloc::LeakingPageAllocator = lol_alloc::LeakingPageAllocator;
+    #[panic_handler]
+    fn panic(_: &core::panic::PanicInfo) -> ! { core::arch::wasm32::unreachable() }
+
+  Two real costs:
+  - lol_alloc had to be listed in every plugin's Cargo.toml. The
+    workspace already needed it for the host compiler; making each
+    plugin add it again is duplication that drifts (different version
+    pins between plugins is a real failure mode).
+  - The #[panic_handler] is wasm32-only. New plugin authors who try
+    to add unit tests find their host build fails because panic_handler
+    is `duplicate language item` or `cannot apply to host build`.
+    The cfg-gating that fixes this isn't obvious; the macro hides it.
+
+Fix:
+  - Added a wasm32-only `lol_alloc = "0.4"` dependency in
+    edge-pdk/Cargo.toml under `[target.'cfg(target_arch = "wasm32")'.dependencies]`,
+    so the host build (e.g. cargo test on the PDK itself) does not pull
+    it in.
+  - Re-exported the crate as `__lol_alloc` (under a hidden path) from
+    edge-pdk/src/lib.rs so the macro can name the symbol regardless of
+    whether the user added their own `lol_alloc` dep.
+  - Added a `macro_rules! module` (#[macro_export]) that emits the
+    #[global_allocator] static and the #[panic_handler] fn under
+    `#[cfg(target_arch = "wasm32")]`, so on host builds the macro
+    expands to nothing and `cargo test` works without further changes.
+
+  Plugin authors can now write `edge_pdk::module!();` once at the top
+  of lib.rs and drop the four boilerplate lines. The pre-existing
+  manual pattern still works; the macro is strictly additive.
+
+  /* ... */ doc comment in the lexer/parser style records the exact
+  attributes the macro emits and the still-required crate-root
+  attributes (#![no_std], #![no_main], extern crate alloc;) that
+  cannot be injected from inside an item position.
+
+Documentation:
+  reference/wasm-abi.md and reference/writing-modules.md show the
+  manual pattern. Both are still valid — the macro is opt-in. No doc
+  updates are strictly required (recommending the macro path is a
+  pure migration suggestion, not a correctness fix).
+
+Build & tests: cargo build -p edge-pdk -> ok; cargo test -> 6/6 passed.

add-wasm-free-export.txt

@@ -0,0 +1,40 @@
+Branch: add-wasm-free-export
+Tech debt: D17 (partial) — wasm_alloc has no symmetric wasm_free, every host alloc leaks.
+
+================================================================================
+Commit 1: add wasm_free counterpart for wasm_alloc
+================================================================================
+File: compiler/src/main/exports.rs
+
+What was wrong:
+  wasm_alloc returned a pointer obtained from Box::into_raw(...) of a
+  boxed slice, but no export let the host hand the pointer back to be
+  reclaimed. Every host-side staging buffer therefore leaked for the
+  lifetime of the WASM instance. The WASM runtime had grown to expect
+  this — the comment in mod.rs about the bump allocator masks the
+  fact that the underlying allocator itself can free, the host just
+  had no surface to ask for it.
+
+Fix:
+  Added the symmetric export:
+
+    pub unsafe extern "C" fn wasm_free(ptr: *mut u8, size: u32);
+
+  It reconstructs the boxed slice from (ptr, size) and drops it — the
+  exact inverse of wasm_alloc's `Box::into_raw(boxed_slice)`. Treats
+  null ptr and size == 0 as no-ops so the host can call it
+  unconditionally after every alloc. A /* ... */ comment in the
+  lexer/parser style records the precondition (size must match the
+  original alloc).
+
+  This does NOT address the broader D2 (deprecate SRC/OUT/INP in
+  favour of handle-based ABI) — that's structural. It just closes
+  the alloc/free asymmetry so a host that wants to release buffers
+  can, today, without changing the rest of the wire.
+
+Documentation:
+  No .md docs reference wasm_alloc or wasm_free at the wire level (the
+  WASM ABI doc covers host imports / exports for handles, not these
+  staging buffers). No doc updates required.
+
+Build & tests: cargo build -p edge-python -> ok; cargo test -> 6/6 passed.

centralize-nanbox-constants.txt

@@ -0,0 +1,75 @@
+Branch: centralize-nanbox-constants
+Tech debt: D20 — NaN-boxing constants existed in three independent copies (classify_encode, classify_decode, vm::types::Val), drifting was a silent ABI break.
+
+================================================================================
+Commit 1: lift NaN-box layout into abi::nan_box
+================================================================================
+Files:
+  - compiler/src/abi.rs
+  - compiler/src/modules/vm/types/mod.rs
+
+What was wrong:
+  Three places defined the same constants:
+
+  1. classify_encode (abi.rs:188+) — inner consts: QNAN, TAG_NONE_BITS,
+     TAG_TRUE_BITS, TAG_FALSE_BITS, TAG_INT_BITS.
+  2. classify_decode (abi.rs:247+) — inner consts: QNAN, SIGN, TAG_INT.
+  3. vm::types::Val (mod.rs:64+) — module-level consts: QNAN, SIGN,
+     TAG_UNDEF, TAG_NONE, TAG_TRUE, TAG_FALSE, TAG_INT, TAG_HEAP.
+
+  All three carried the same numeric values, but cargo had no way to
+  prove that. A bump in any one (e.g. moving the heap tag bit nibble)
+  would compile cleanly and silently corrupt round-tripping at the
+  wire boundary. The doc-comments openly admitted the duplication
+  ("must match host Val impl", "Same NaN-boxing constants as
+  classify_encode") without proposing a remedy.
+
+  The 47-bit int payload mask (0x0000_FFFF_FFFF_FFFF) and the 28-bit
+  heap-index mask (0x0FFF_FFFF) were also embedded inline at the use
+  sites without symbolic names.
+
+Fix:
+  Added a sealed `pub mod nan_box` at the top of abi.rs holding every
+  NaN-boxing constant in a single source-of-truth:
+
+    pub mod nan_box {
+        pub const QNAN: u64        = 0x7FFC_0000_0000_0000;
+        pub const SIGN: u64        = 0x8000_0000_0000_0000;
+        pub const TAG_UNDEF: u64   = QNAN;
+        pub const TAG_NONE: u64    = QNAN | 1;
+        pub const TAG_TRUE: u64    = QNAN | 2;
+        pub const TAG_FALSE: u64   = QNAN | 3;
+        pub const TAG_INT: u64     = QNAN | SIGN;
+        pub const TAG_HEAP: u64    = QNAN | 4;
+        pub const INT_PAYLOAD_MASK: u64 = 0x0000_FFFF_FFFF_FFFF;
+    }
+
+  - classify_encode now `use nan_box::*;` and references TAG_NONE /
+    TAG_TRUE / TAG_FALSE / TAG_INT / INT_PAYLOAD_MASK directly. Two
+    of the locally-defined names lost their `_BITS` suffix because
+    the canonical names already convey "bit pattern" through context.
+  - classify_decode same treatment, plus the magic 0x0000_FFFF_FFFF_FFFF
+    now reads INT_PAYLOAD_MASK and the QNAN|1/2/3 literals collapse to
+    TAG_NONE/TAG_TRUE/TAG_FALSE.
+  - vm::types::Val replaced its eight module-level `const` declarations
+    with `use crate::abi::nan_box::{...}`. The two inline mask
+    occurrences inside `Val::int` and `Val::as_int` now name
+    INT_PAYLOAD_MASK explicitly.
+  - 28-bit heap index mask (0x0FFF_FFFF) left inline since it appears
+    once and is structurally bound to the `>> 4` shift in the same
+    expression.
+
+  /* ... */ doc comment in the lexer/parser style on the new module
+  records the sealing rule (any layout change is an ABI bump, ties
+  into D22's version handshake).
+
+  Numeric values are byte-identical to before. Tests prove the
+  layout still round-trips through the VM and the wire codec.
+
+Documentation:
+  README.md and design.md describe the NaN-box layout at the abstract
+  level (e.g. "Int = QNAN | SIGN | i47"). The descriptions remain
+  accurate; the file-path internals are not user-visible. No doc
+  updates required.
+
+Build & tests: cargo build -p edge-python -> ok; cargo test -> 6/6 passed.

centralize-src-buffer-read.txt

@@ -0,0 +1,47 @@
+Branch: centralize-src-buffer-read
+Tech debt: D17 / D62 — extract_imports and run duplicated SRC reading with divergent error handling.
+
+================================================================================
+Commit 1: extract read_src helper for SRC buffer
+================================================================================
+File: compiler/src/main/exports.rs
+
+What was wrong:
+  Both extern "C" entry points opened the host-owned SRC buffer with the
+  same five-line preamble:
+
+    let len = len.min(SZ);
+    let src = match core::str::from_utf8(unsafe {
+        core::slice::from_raw_parts(core::ptr::addr_of!(SRC) as *const u8, len)
+    }) {
+        Ok(s) => s,
+        Err(_) => ...
+    };
+
+  The cap-and-validate logic was identical, but the error paths
+  diverged: extract_imports silently returned write_out(""), while run
+  formatted "input rejected: invalid utf-8 at byte N". Adding a third
+  caller would have spawned a third dialect of the same boundary check.
+
+Fix:
+  Introduced a single private helper:
+
+    unsafe fn read_src(len: usize) -> Result<&'static str, Utf8Error>
+
+  that performs the cap-and-validate once and returns the &'static str
+  view (SRC is `static`, so the slice's lifetime is genuinely 'static).
+
+  Both callers now match on the Result and decide how to surface the
+  failure: extract_imports keeps its silent-empty contract, run keeps
+  its user-facing error. The behaviour is byte-identical to before;
+  only the duplication is gone.
+
+  A new caller now has one place to look and one decision to make
+  ("silent or formatted error?"), not five duplicated lines to copy.
+
+Documentation:
+  No .md docs describe the boundary handling at this granularity (the
+  mentions of `run()` in design.md / language docs refer to Python's
+  builtin `run()`, not the WASM export). No doc updates required.
+
+Build & tests: cargo build -p edge-python -> ok; cargo test -> 6/6 passed.

collapse-push-macro-into-s.txt

@@ -0,0 +1,35 @@
+Branch: collapse-push-macro-into-s
+Tech debt: D43 — redundant push! macro alongside s!.
+
+================================================================================
+Commit 1: drop unused push macro
+================================================================================
+Files:
+  - compiler/src/modules/fstr.rs
+  - documentation/implementation/design.md
+
+What was wrong:
+  fstr.rs exported two #[macro_export] macros that covered the same
+  domain (no-alloc string formatting):
+    - push!(s, ...) — single fragment, six explicit type arms.
+    - s!(...) — multiple fragments via a recursive @b helper, same
+      type matrix.
+  Any new fragment kind (e.g. hex, dec_groups) had to be added in two
+  places, with mechanical drift risk. push! also lacked s!'s `cap:` form
+  and could not be chained.
+
+  Workspace-wide grep for `push!` returned zero call sites outside the
+  declaration itself. The macro was simply dead surface.
+
+Fix:
+  Removed the entire push! definition. s! remains the single string
+  builder. No call sites had to change.
+
+Documentation:
+  documentation/implementation/design.md line 92 listed
+  `# numeric formatter + s!/push!/err! string macros` for fstr.rs.
+  Updated to drop `push!/` so the inventory matches the file.
+
+  compiler/README.md only lists the file path; no macro names to drift.
+
+Tests: cargo test -> 6/6 passed.

compiler/Cargo.toml

@@ -20,18 +20,18 @@ name = "tests"
 path = "tests/main.rs"
 
 [dependencies]
-hashbrown = { version = "0.17", default-features = false }
-itoa = "1"
+hashbrown = { workspace = true }
+itoa = { workspace = true }
 
 # WASM-only global allocator. Pulled into wasm32 builds automatically; absent
 # on host builds (where std's allocator is fine and `compiler.wasm` is just a
 # library artifact, not the runtime target).
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-lol_alloc = "0.4"
+lol_alloc = { workspace = true }
 
 # Test-only deps for the JSON-driven test runner. None of these are pulled
 # into the release `compiler.wasm`.
 [dev-dependencies]
-hashbrown = { version = "0.17", default-features = false, features = ["serde"] }
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
+hashbrown = { workspace = true, features = ["serde"] }
+serde = { workspace = true }
+serde_json = { workspace = true }