Updated with persistance in the index HNSW

Author: anaslimem

Cargo.lock

@@ -2,14 +2,21 @@
 
 [![License: MIT/Apache-2.0](https://img.shields.io/badge/License-MIT%2FApache--2.0-blue.svg)](LICENSE)
 [![Status: Beta](https://img.shields.io/badge/Status-Beta-brightgreen.svg)](#current-status)
-[![Version](https://img.shields.io/badge/Version-0.1.2-blue.svg)](https://github.com/anaslimem/CortexaDB/releases)
+[![Version](https://img.shields.io/badge/Version-0.1.3-blue.svg)](https://github.com/anaslimem/CortexaDB/releases)
 
 **CortexaDB** is a simple, fast, and hard-durable embedded database designed specifically for AI agent memory. It provides a single-file-like experience (no server required) but with native support for vectors, graphs, and temporal search.
 
 Think of it as **SQLite, but with semantic and relational intelligence for your agents.**
 
 ---
 
+## What's New in v0.1.3
+
+- **Automatic HNSW Persistence** - HNSW index is now automatically saved to disk on checkpoint or database close, enabling fast restart without rebuilding the index
+- Improved reliability for production use
+
+---
+
 ## Quickstart
 
 ### Python (Recommended)
@@ -89,6 +96,16 @@ CortexaDB uses **USearch** for high-performance approximate nearest neighbor sea
 | `exact` | Small datasets (<10K) | 100% | O(n) |
 | `hnsw` | Large datasets | 95%+ | O(log n) |
 
+### Automatic Persistence
+
+HNSW indexing now includes **automatic persistence**:
+
+- On `checkpoint()` - HNSW index is saved to disk
+- On database close/drop - HNSW index is automatically saved
+- On restart - HNSW index is loaded from disk (fast recovery!)
+
+No extra configuration needed - just use `index_mode="hnsw"` and it just works.
+
 ```python
 from cortexadb import CortexaDB, HashEmbedder
 
@@ -218,7 +235,7 @@ We use a custom versioned serialization layer (with a "magic-byte" header). This
 ---
 
 ## License & Status
-CortexaDB is currently in **Beta (v0.1.2)**. It is released under the **MIT** and **Apache-2.0** licenses.  
+CortexaDB is currently in **Beta (v0.1.3)**. It is released under the **MIT** and **Apache-2.0** licenses.  
 We are actively refining the API and welcome feedback!
 
 ---

README.md

@@ -1,6 +1,6 @@
 [package]
 name = "cortexadb-core"
-version = "0.1.2"
+version = "0.1.3"
 edition = "2024"
 authors = ["Anas Limem <limemanas0@gmail.com>"]
 description = "Fast, embedded vector + graph memory for AI agents"

crates/cortexadb-core/Cargo.toml

@@ -4,7 +4,7 @@ use thiserror::Error;
 use crate::core::memory_entry::MemoryId;
 use crate::core::state_machine::StateMachine;
 use crate::index::graph::GraphIndex;
-use crate::index::hnsw::HnswConfig;
+use crate::index::hnsw::{HnswBackend, HnswConfig};
 use crate::index::temporal::TemporalIndex;
 use crate::index::vector::{VectorBackendMode, VectorIndex};
 
@@ -78,12 +78,22 @@ impl IndexLayer {
         Self { vector: VectorIndex::new(vector_dimension) }
     }
 
-    /// Create new index layer with HNSW enabled
+    /// Create new index layer with HNSW enabled (fresh build)
     pub fn new_with_hnsw(vector_dimension: usize, hnsw_config: HnswConfig) -> Self {
-        let vector = match VectorIndex::new_with_hnsw(vector_dimension, hnsw_config) {
-            Ok(v) => v,
-            Err(_) => VectorIndex::new(vector_dimension),
-        };
+        Self::new_with_loaded_hnsw(vector_dimension, hnsw_config, None)
+    }
+
+    /// Create new index layer with optional pre-loaded HNSW backend
+    pub fn new_with_loaded_hnsw(
+        vector_dimension: usize,
+        hnsw_config: HnswConfig,
+        loaded_hnsw: Option<HnswBackend>,
+    ) -> Self {
+        let vector =
+            match VectorIndex::new_with_loaded_hnsw(vector_dimension, hnsw_config, loaded_hnsw) {
+                Ok(v) => v,
+                Err(_) => VectorIndex::new(vector_dimension),
+            };
         Self { vector }
     }
 

crates/cortexadb-core/src/index/combined.rs

@@ -1,3 +1,4 @@
+use std::path::Path;
 use std::sync::{Arc, Mutex};
 use thiserror::Error;
 
@@ -142,4 +143,47 @@ impl HnswBackend {
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
+
+    pub fn save_to_file(&self, path: &Path) -> Result<()> {
+        let index = self.index.lock().map_err(|_| HnswError::LockError)?;
+
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+
+        let path_str = path.to_string_lossy().to_string();
+        index.save(&path_str).map_err(|e| HnswError::UsearchError(e.to_string()))?;
+        Ok(())
+    }
+
+    pub fn load_from_file(path: &Path, dimension: usize, config: HnswConfig) -> Result<Self> {
+        if !path.exists() {
+            return Err(HnswError::IoError(std::io::Error::new(
+                std::io::ErrorKind::NotFound,
+                "HNSW index file not found",
+            )));
+        }
+
+        let options = usearch::IndexOptions {
+            dimensions: dimension,
+            metric: usearch::MetricKind::Cos,
+            quantization: usearch::ScalarKind::F32,
+            connectivity: config.m,
+            expansion_add: config.ef_construction,
+            expansion_search: config.ef_search,
+            ..Default::default()
+        };
+
+        let index =
+            usearch::new_index(&options).map_err(|e| HnswError::UsearchError(e.to_string()))?;
+
+        let path_str = path.to_string_lossy().to_string();
+        index.load(&path_str).map_err(|e| HnswError::UsearchError(e.to_string()))?;
+
+        Ok(Self { index: Arc::new(Mutex::new(index)), dimension, config })
+    }
+
+    pub fn dimension(&self) -> usize {
+        self.dimension
+    }
 }

crates/cortexadb-core/src/index/hnsw.rs

@@ -212,18 +212,33 @@ impl VectorIndex {
         }
     }
 
-    /// Create a new vector index with HNSW enabled
+    /// Create a new vector index with HNSW enabled (fresh build)
     pub fn new_with_hnsw(vector_dimension: usize, config: HnswConfig) -> Result<Self> {
-        let hnsw_backend =
-            HnswBackend::new(vector_dimension, config).map_err(|_e| VectorError::NoEmbeddings)?;
+        Self::new_with_loaded_hnsw(vector_dimension, config, None)
+    }
+
+    /// Create a new vector index with optional pre-loaded HNSW backend
+    pub fn new_with_loaded_hnsw(
+        vector_dimension: usize,
+        config: HnswConfig,
+        loaded_hnsw: Option<HnswBackend>,
+    ) -> Result<Self> {
+        let hnsw_backend = match loaded_hnsw {
+            Some(backend) => Some(Arc::new(backend)),
+            None => {
+                let backend = HnswBackend::new(vector_dimension, config)
+                    .map_err(|_e| VectorError::NoEmbeddings)?;
+                Some(Arc::new(backend))
+            }
+        };
         Ok(Self {
             partitions: HashMap::new(),
             id_to_namespace: HashMap::new(),
             vector_dimension,
             backend_mode: VectorBackendMode::Exact,
             backend: Arc::new(ExactBackend),
             ann_provider: Arc::new(PrefixAnnCandidateProvider),
-            hnsw_backend: Some(Arc::new(hnsw_backend)),
+            hnsw_backend,
         })
     }
 
@@ -255,6 +270,31 @@ impl VectorIndex {
         self.hnsw_backend.is_some()
     }
 
+    /// Save HNSW index to disk (no-op if HNSW not enabled)
+    pub fn save_hnsw(&self, path: &std::path::Path) -> std::io::Result<()> {
+        if let Some(ref hnsw) = self.hnsw_backend {
+            hnsw.save_to_file(path)
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+        }
+        Ok(())
+    }
+
+    /// Load HNSW index from disk (returns None if file doesn't exist)
+    pub fn load_hnsw(
+        path: &std::path::Path,
+        dimension: usize,
+        config: HnswConfig,
+    ) -> std::io::Result<Option<HnswBackend>> {
+        if !path.exists() {
+            return Ok(None);
+        }
+
+        match HnswBackend::load_from_file(path, dimension, config) {
+            Ok(backend) => Ok(Some(backend)),
+            Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())),
+        }
+    }
+
     pub fn backend_mode(&self) -> VectorBackendMode {
         self.backend_mode
     }

crates/cortexadb-core/src/index/vector.rs

@@ -103,6 +103,7 @@ pub struct CortexaDBStore {
     sync_thread: Option<JoinHandle<()>>,
     checkpoint_policy: CheckpointPolicy,
     checkpoint_path: std::path::PathBuf,
+    hnsw_path: std::path::PathBuf,
     checkpoint_control: Arc<(Mutex<CheckpointRuntime>, Condvar)>,
     checkpoint_thread: Option<JoinHandle<()>>,
     capacity_policy: CapacityPolicy,
@@ -243,14 +244,38 @@ impl CortexaDBStore {
         checkpoint_path: std::path::PathBuf,
         index_mode: crate::index::hnsw::IndexMode,
     ) -> Result<Self> {
+        let hnsw_path = checkpoint_path.with_extension("hnsw");
+
         let hnsw_config = match index_mode {
             crate::index::hnsw::IndexMode::Exact => None,
             crate::index::hnsw::IndexMode::Hnsw(config) => Some(config),
         };
+
+        let loaded_hnsw = if let Some(config) = hnsw_config.as_ref() {
+            match crate::index::VectorIndex::load_hnsw(&hnsw_path, vector_dimension, config.clone())
+            {
+                Ok(Some(backend)) => {
+                    eprintln!("Loaded HNSW index from disk (fast recovery)");
+                    Some(backend)
+                }
+                Ok(None) => {
+                    eprintln!("No HNSW index file found, building fresh index");
+                    None
+                }
+                Err(e) => {
+                    eprintln!("Failed to load HNSW index, rebuilding: {}", e);
+                    None
+                }
+            }
+        } else {
+            None
+        };
+
         let indexes = Self::build_vector_index(
             engine.get_state_machine(),
             vector_dimension,
             hnsw_config.as_ref(),
+            loaded_hnsw,
         )?;
         Self::assert_vector_index_in_sync_inner(engine.get_state_machine(), &indexes)?;
 
@@ -297,6 +322,7 @@ impl CortexaDBStore {
             sync_thread,
             checkpoint_policy,
             checkpoint_path,
+            hnsw_path,
             checkpoint_control,
             checkpoint_thread,
             capacity_policy,
@@ -320,6 +346,10 @@ impl CortexaDBStore {
         let last_applied_id = writer.engine.last_applied_id().0;
         save_checkpoint(&self.checkpoint_path, snapshot.state_machine(), last_applied_id)?;
 
+        if let Err(e) = snapshot.indexes.vector_index().save_hnsw(&self.hnsw_path) {
+            eprintln!("Warning: Failed to save HNSW index: {}", e);
+        }
+
         // Truncate WAL prefix — only keep entries written after the checkpoint.
         let wal_path = writer.engine.wal_path().to_path_buf();
         WriteAheadLog::truncate_prefix(&wal_path, CommandId(last_applied_id))?;
@@ -515,6 +545,7 @@ impl CortexaDBStore {
             writer.engine.get_state_machine(),
             writer.indexes.vector.dimension(),
             None,
+            None,
         )?;
 
         let indexed = writer.indexes.vector.len();
@@ -786,12 +817,25 @@ impl CortexaDBStore {
         state_machine: &StateMachine,
         vector_dimension: usize,
         hnsw_config: Option<&crate::index::hnsw::HnswConfig>,
+        loaded_hnsw: Option<crate::index::hnsw::HnswBackend>,
     ) -> Result<IndexLayer> {
+        let has_loaded_hnsw = loaded_hnsw.is_some();
         let indexes = match hnsw_config {
-            Some(config) => IndexLayer::new_with_hnsw(vector_dimension, config.clone()),
+            Some(config) => {
+                if let Some(loaded) = loaded_hnsw {
+                    IndexLayer::new_with_loaded_hnsw(vector_dimension, config.clone(), Some(loaded))
+                } else {
+                    IndexLayer::new_with_hnsw(vector_dimension, config.clone())
+                }
+            }
             None => IndexLayer::new(vector_dimension),
         };
         let mut indexes = indexes;
+
+        if has_loaded_hnsw {
+            return Ok(indexes);
+        }
+
         for entry in state_machine.all_memories() {
             if let Some(embedding) = entry.embedding.clone() {
                 indexes.vector_index_mut().index_in_namespace(
@@ -855,6 +899,12 @@ impl Drop for CortexaDBStore {
         if self.checkpoint_policy != CheckpointPolicy::Disabled {
             let _ = self.checkpoint_now();
         }
+
+        // Always save HNSW index on drop if it exists (automatic persistence)
+        let snapshot = self.snapshot.load_full();
+        if let Err(e) = snapshot.indexes.vector_index().save_hnsw(&self.hnsw_path) {
+            eprintln!("Warning: Failed to save HNSW on drop: {}", e);
+        }
     }
 }
 

crates/cortexadb-core/src/store.rs

@@ -1,6 +1,6 @@
 [package]
 name = "cortexadb-py"
-version = "0.1.0"
+version = "0.1.3"
 edition = "2024"
 
 [lib]

crates/cortexadb-py/Cargo.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "cortexadb"
-version = "0.1.2"
+version = "0.1.3"
 requires-python = ">=3.9"
 description = "Fast, embedded vector + graph memory for AI agents"
 authors = [