Introduced configurable write durability for Mnemos databases

Author: anaslimem

crates/mnemos-py/mnemos/client.py

@@ -132,10 +132,11 @@ def __init__(
         path: str,
         dimension: t.Optional[int],
         embedder: t.Optional[Embedder] = None,
+        sync: str = "strict",
     ):
         self._embedder = embedder
         try:
-            self._inner = _mnemos.Mnemos.open(path, dimension=dimension)
+            self._inner = _mnemos.Mnemos.open(path, dimension=dimension, sync=sync)
         except Exception as e:
             raise MnemosError(str(e))
 
@@ -146,6 +147,7 @@ def open(
         *,
         dimension: t.Optional[int] = None,
         embedder: t.Optional[Embedder] = None,
+        sync: str = "strict",
     ) -> "Mnemos":
         """
         Open or create a Mnemos database.
@@ -158,6 +160,15 @@ def open(
                        pre-computed embeddings.
             embedder:  An :class:`~mnemos.Embedder` instance. The dimension is
                        inferred from ``embedder.dimension`` automatically.
+            sync:      Write durability policy. One of:
+
+                       * ``"strict"`` *(default)* — every write is fsynced immediately.
+                         Safe against crashes, slightly lower throughput.
+                       * ``"async"`` — writes are fsynced every 10 ms in the
+                         background. Higher throughput but may lose the last few
+                         writes on unclean shutdown.
+                       * ``"batch"`` — fsync every 64 writes or 50 ms, whichever
+                         comes first. A middle ground.
 
         Raises:
             MnemosError: If neither or both of *dimension* and *embedder* are
@@ -173,7 +184,7 @@ def open(
             )
 
         dim = embedder.dimension if embedder is not None else dimension
-        return cls(path, dimension=dim, embedder=embedder)
+        return cls(path, dimension=dim, embedder=embedder, sync=sync)
 
     # ------------------------------------------------------------------
     # Internal helpers

crates/mnemos-py/src/lib.rs

@@ -167,17 +167,28 @@ impl PyMnemos {
     ///     MnemosError: If the database cannot be opened or the dimension
     ///         mismatches an existing database.
     #[staticmethod]
-    #[pyo3(text_signature = "(path, *, dimension)")]
-    fn open(path: &str, dimension: usize) -> PyResult<Self> {
+    #[pyo3(
+        text_signature = "(path, *, dimension, sync='strict')",
+        signature = (path, *, dimension, sync="strict".to_string())
+    )]
+    fn open(path: &str, dimension: usize, sync: String) -> PyResult<Self> {
         if dimension == 0 {
             return Err(MnemosError::new_err("dimension must be > 0"));
         }
 
+        let sync_policy = match sync.to_lowercase().as_str() {
+            "strict" => SyncPolicy::Strict,
+            "async"  => SyncPolicy::Async { interval_ms: 10 },
+            "batch"  => SyncPolicy::Batch { max_ops: 64, max_delay_ms: 50 },
+            other    => return Err(MnemosError::new_err(format!(
+                "unknown sync policy '{}'. Valid values: 'strict', 'async', 'batch'",
+                other,
+            ))),
+        };
+
         let config = facade::MnemosConfig {
             vector_dimension: dimension,
-            sync_policy: SyncPolicy::Async {
-                interval_ms: 10,
-            },
+            sync_policy,
             checkpoint_policy: CheckpointPolicy::Periodic {
                 every_ops: 1000,
                 every_ms: 30_000,

crates/mnemos-py/test_stress.py

@@ -14,40 +14,39 @@ def clean_db_path(request):
         shutil.rmtree(db_path)
 
 def test_replay_safety(clean_db_path):
-    print("\n--- Test 1: Replay Safety (50k inserts) ---")
-    
-    with Mnemos.open(clean_db_path, dimension=2) as db:
+    print("\n--- Test 1: Replay Safety (5000 inserts) ---")
+
+    # Use strict sync so every write is fsynced — guarantees full replay on reopen.
+    with Mnemos.open(clean_db_path, dimension=2, sync="strict") as db:
         start_time = time.time()
         for i in range(5000):
             db.remember(f"Entry {i}", embedding=[0.5, 0.5])
-        
+
         print(f"Inserted 5,000 memories in {time.time() - start_time:.2f}s")
         assert len(db) == 5000
 
-    # Simulate closing / crash then reopen
+    # Simulate closing then reopen — strict policy guarantees no data loss.
     print("Reopening...")
-    with Mnemos.open(clean_db_path, dimension=2) as db2:
-        # With Async sync policy a few trailing entries may not be flushed
-        # before the context manager drops the handle — allow up to 20 missing.
-        assert len(db2) >= 4980, f"Expected ~5000 entries after reopen, got {len(db2)}"
-    
+    with Mnemos.open(clean_db_path, dimension=2, sync="strict") as db2:
+        assert len(db2) == 5000, f"Expected 5000 entries after reopen, got {len(db2)}"
+
     print("Test 1 PASS")
 
 def test_compaction_integrity(clean_db_path):
     print("\n--- Test 3: WAL Compaction Integrity ---")
-    
-    with Mnemos.open(clean_db_path, dimension=2) as db:
-        # Insert 100 entries
+
+    # Use strict sync so compact sees all 100 entries in the WAL.
+    with Mnemos.open(clean_db_path, dimension=2, sync="strict") as db:
         for _ in range(100):
             db.remember("Stress entry", embedding=[0.1, 0.9])
-        
+
         assert len(db) == 100
 
         print("Compacting...")
         db.compact()
 
     print("Reopening...")
-    with Mnemos.open(clean_db_path, dimension=2) as db2:
-        assert len(db2) >= 90, f"Expected ~100 entries after compact + reopen, got {len(db2)}"
-        
+    with Mnemos.open(clean_db_path, dimension=2, sync="strict") as db2:
+        assert len(db2) == 100, f"Expected 100 entries after compact + reopen, got {len(db2)}"
+
     print("Test 3 PASS")