feat: Implement graph-first memory ingestion and reasoning pipeline

- Added `ingestion.py` for parsing text and binding graph structure to memories.
- Introduced `reasoning.py` for multi-hop memory inference and relation path explanation.
- Created `scheduler.py` to manage memory tier transitions based on access patterns.
- Developed `self_tune.py` for online tuning of retrieval weights based on agent feedback.
- Implemented `fusion.py` for multi-objective retrieval scoring and weight normalization.
- Added `ranker.py` for unified retrieval ranking and explainability.
- Integrated `crewai_adapter.py` for CrewAI compatibility with OMem.
- Developed `llama_index.py` for LlamaIndex integration as a retrieval backend.
- Created tests for graph substrate and retrieval functionalities.

Author: mohitkumarrajbadi

omem/__init__.py

@@ -3,11 +3,16 @@
 from .api import OMem
 from .core.engine import DreamResult, ForgetResult
 from .types import (
+    Evidence,
+    GraphNode,
     Memory,
+    MemoryLevel,
     MemoryPriority,
     MemoryStatus,
     MemoryTier,
     MemoryType,
+    Provenance,
+    RelationEdge,
     RetrievalExplanation,
 )
 
@@ -22,6 +27,11 @@
     "MemoryPriority",
     "MemoryStatus",
     "Memory",
+    "MemoryLevel",
+    "GraphNode",
+    "RelationEdge",
+    "Evidence",
+    "Provenance",
     "RetrievalExplanation",
     "ForgetResult",
     "DreamResult",

omem/api.py

@@ -11,7 +11,7 @@
 from .backends.sqlite import SQLiteBackend
 from .core.engine import BrainTrace, DreamResult, ForgetResult
 from .security.audit import AuditLogger
-from .types import Memory, MemoryType, RetrievalExplanation
+from .types import Memory, MemoryTier, MemoryType, RetrievalExplanation
 
 logger = logging.getLogger(__name__)
 
@@ -101,6 +101,57 @@ def add(
         self._audit.log("add", memory_id=memory_id_result, namespace=namespace, source=source)
         return memory_id_result
 
+    def add_experience(
+        self,
+        content: str,
+        namespace: str = "default",
+        source: str = "experience",
+        confidence: float = 1.0,
+        importance: Optional[float] = None,
+        metadata: Optional[Dict] = None,
+    ) -> str:
+        """Graph-first ingestion for unstructured experience text."""
+        memory_id = self.brain.add_experience(
+            content,
+            namespace=namespace,
+            source=source,
+            confidence=confidence,
+            importance=importance,
+            metadata=metadata,
+        )
+        self._audit.log("add_experience", memory_id=memory_id, namespace=namespace)
+        return memory_id
+
+    def link_entities(
+        self,
+        source: str,
+        target: str,
+        relation: str = "related_to",
+        memory_id: str = "",
+        confidence: float = 1.0,
+    ) -> str:
+        """Explicitly link two entities in the knowledge graph."""
+        return self.brain.link_entities(
+            source, target, relation, memory_id=memory_id, confidence=confidence
+        )
+
+    def assert_fact(
+        self,
+        subject: str,
+        relation: str,
+        obj: str,
+        memory_id: str = "",
+        confidence: float = 0.9,
+    ) -> Dict:
+        """Assert a structured fact as a high-confidence graph relation."""
+        return self.brain.assert_fact(
+            subject, relation, obj, memory_id=memory_id, confidence=confidence
+        )
+
+    def query_graph(self, entity_name: str, depth: int = 2) -> Dict:
+        """Structured graph query: nodes, edges, and related memory IDs."""
+        return self.brain.query_graph(entity_name, depth=depth)
+
     def add_batch(
         self,
         contents: List[str],
@@ -143,16 +194,27 @@ def recall(
         time_range: Optional[str] = None,
         namespace: Optional[str] = None,
         project_only: bool = False,
+        level: Optional[str] = None,
+        tiers: Optional[List[MemoryTier]] = None,
+        explain: bool = False,
+        weight_overrides: Optional[Dict] = None,
+        include_archive: bool = False,
     ) -> List[Memory]:
         """Advanced retrieval with context-type boosting and temporal filtering.
 
         Args:
             query: The search query.
             k: Number of results.
             context_type: One of 'architecture', 'bugs', 'decisions', etc.
+            mode: Retrieval mode profile ('default', 'planning', 'coding', 'chat', 'recall').
             time_range: One of 'today', 'recent', 'last_week'.
             namespace: Specific namespace.
             project_only: If True, only searches the provided namespace (doesn't mix global).
+            level: Hierarchy filter ('working', 'short_term', 'long_term', 'archive').
+            tiers: Optional list of MemoryTier enums to restrict search.
+            explain: If True, populate explanations via get_explanations().
+            weight_overrides: Optional dict to override fusion weight components.
+            include_archive: Include archived memories in search.
         """
         import time
 
@@ -231,7 +293,17 @@ def recall(
                 )
 
         results = self.brain.rag(
-            query, top_k=top_k, namespace=search_namespace, type_boosts=type_boosts
+            query,
+            top_k=top_k,
+            namespace=search_namespace,
+            type_boosts=type_boosts,
+            mode=mode or "default",
+            level=level,
+            tiers=tiers,
+            explain=explain,
+            weight_overrides=weight_overrides,
+            include_archive=include_archive,
+            include_inactive=include_archive,
         )
 
         # Post-filter for namespace if we searched wide
@@ -323,9 +395,33 @@ def inspect(
         query: str,
         top_k: int = 5,
         namespace: Optional[str] = None,
+        mode: str = "default",
+        weight_overrides: Optional[Dict] = None,
     ) -> List[RetrievalExplanation]:
-        """Explain the scoring for a given query."""
-        return self.brain.inspect(query, top_k, namespace)
+        """Explain the fusion scoring breakdown for a given query."""
+        return self.brain.inspect(
+            query,
+            top_k,
+            namespace,
+            mode=mode,
+            weight_overrides=weight_overrides,
+        )
+
+    def get_explanations(self) -> List[RetrievalExplanation]:
+        """Return explanations from the last explain=True recall."""
+        return self.brain.get_last_explanations()
+
+    def set_fusion_weights(self, weights: Dict[str, float]) -> None:
+        """Set default fusion weights for retrieval scoring."""
+        from .core.retrieval.fusion import FusionWeights
+
+        current = self.brain.get_fusion_weights().as_dict()
+        current.update(weights)
+        self.brain.set_fusion_weights(FusionWeights(**current))
+
+    def get_fusion_weights(self) -> Dict[str, float]:
+        """Return current fusion weight configuration."""
+        return self.brain.get_fusion_weights().as_dict()
 
     def namespace_stats(self, namespace: str) -> Dict:
         """Get stats for a specific namespace."""

omem/codebase/retriever.py

@@ -13,14 +13,10 @@
 from typing import Any, Dict, List
 
 from ..api import OMem
+from ..core.retrieval.fusion import fuse_score
+from ..core.retrieval.ranker import weights_for_mode
 
-# Default ranking weights – these can be tuned later via a config file.
-DEFAULT_WEIGHTS = {
-    "semantic": 0.5,
-    "graph": 0.2,
-    "recency": 0.15,
-    "importance": 0.15,
-}
+DEFAULT_WEIGHTS = weights_for_mode("coding").as_dict()
 
 class CodeRetriever:
     """Retrieve code symbols for a natural‑language query.
@@ -104,10 +100,13 @@ def retrieve(
         # Pick the primary result (highest combined weight without graph yet)
         def combined(s):
             w = self.weights
-            return (
-                w["semantic"] * s["semantic"]
-                + w["recency"] * s["recency"]
-                + w["importance"] * s["importance"]
+            return fuse_score(
+                semantic=s["semantic"],
+                keyword=0.0,
+                recency=s["recency"],
+                importance=s["importance"],
+                graph=s["graph"],
+                weights=weights_for_mode("coding"),
             )
 
         scored.sort(key=combined, reverse=True)
@@ -120,11 +119,13 @@ def combined(s):
                 entry["graph"] = 1.0 / (1 + self._graph_hops(primary_id, entry["mem"].id))
             # Re‑rank with graph component
             primary.sort(
-                key=lambda s: (
-                    self.weights["semantic"] * s["semantic"]
-                    + self.weights["graph"] * s["graph"]
-                    + self.weights["recency"] * s["recency"]
-                    + self.weights["importance"] * s["importance"]
+                key=lambda s: fuse_score(
+                    semantic=s["semantic"],
+                    keyword=0.0,
+                    recency=s["recency"],
+                    importance=s["importance"],
+                    graph=s["graph"],
+                    weights=weights_for_mode("coding"),
                 ),
                 reverse=True,
             )

omem/core/brain/dream.py

@@ -259,9 +259,13 @@ def dream_consolidate(
             metadata={
                 "source_count": len(cluster),
                 "cluster_types": list(set(m.type.name for m in cluster)),
+                "abstract": True,
             },
             insight_sources=cluster_source_ids,
             consolidation_count=len(cluster),
+            confidence_score=min(0.6 + len(cluster) * 0.05, 0.95),
+            evidence_count=len(cluster),
+            level="long_term",
         )
         insight_memories.append(insight)
         result.insight_ids.append(insight_id)

omem/core/brain/forgetting.py

@@ -83,7 +83,10 @@ def compute_health(memory: Memory, now: Optional[float] = None) -> float:
     utility = getattr(memory, "utility_score", 0.0)
     utility_factor = 1.0 + min(utility, 1.0) * (_MAX_UTILITY_BOOST - 1.0)
 
-    return base * recency * usage * utility_factor
+    confidence_factor = 0.5 + 0.5 * getattr(memory, "confidence_score", 1.0)
+    evidence_factor = min(1.0 + getattr(memory, "evidence_count", 1) * 0.05, 1.5)
+
+    return base * recency * usage * utility_factor * confidence_factor * evidence_factor
 
 
 @dataclass

omem/core/brain/importance.py

@@ -356,3 +356,23 @@ def run_decay_sweep(memories: List[Memory], now: Optional[float] = None) -> List
             mem.active = False
             deactivated.append(mem.id)
     return deactivated
+
+
+def reinforce_on_access(memory: Memory, now: Optional[float] = None) -> None:
+    """Online learning signal: boost importance and confidence on successful retrieval."""
+    now = now or time.time()
+    memory.access_count += 1
+    memory.last_accessed = now
+    memory.freshness = now
+
+    # Diminishing boost — frequently accessed memories stabilize
+    boost = 0.02 / (1.0 + memory.access_count * 0.1)
+    memory.importance = min(memory.importance + boost, 1.0)
+    memory.base_score = min(memory.base_score + boost * 0.5, 2.0)
+    memory.confidence_score = min(
+        compute_confidence_score(memory) + boost * 0.5, 1.0
+    )
+
+    # Promote working → short_term on repeated access
+    if memory.level == "working" and memory.access_count >= 2:
+        memory.level = "short_term"

omem/core/brain/ingestion.py

@@ -0,0 +1,65 @@
+"""Graph-first memory ingestion pipeline.
+
+Experience → Understand → Connect
+
+Parses text, extracts entities/relations, and binds graph structure to memories.
+"""
+
+from dataclasses import dataclass, field
+from typing import List
+
+from ...types import Memory
+from ..graph.knowledge import KnowledgeGraph
+
+
+@dataclass
+class IngestResult:
+    """Structured output from graph-first ingestion."""
+
+    memory_id: str
+    node_ids: List[str] = field(default_factory=list)
+    edge_ids: List[str] = field(default_factory=list)
+    entities: List[str] = field(default_factory=list)
+    confidence: float = 1.0
+    evidence_count: int = 1
+    relation_types: List[str] = field(default_factory=list)
+
+
+def ingest_experience(
+    graph: KnowledgeGraph,
+    memory_id: str,
+    content: str,
+    source: str = "user",
+    confidence: float = 1.0,
+    namespace: str = "default",
+    user_name: str = "",
+) -> IngestResult:
+    """Run the graph ingestion pipeline for a new memory."""
+    payload = graph.ingest_experience(
+        memory_id=memory_id,
+        content=content,
+        source=source,
+        confidence=confidence,
+        namespace=namespace,
+        user_name=user_name,
+    )
+    return IngestResult(
+        memory_id=memory_id,
+        node_ids=payload.get("node_ids", []),
+        edge_ids=payload.get("edge_ids", []),
+        entities=payload.get("entities", []),
+        confidence=payload.get("confidence", confidence),
+        evidence_count=payload.get("evidence_count", 1),
+        relation_types=payload.get("relation_types", []),
+    )
+
+
+def apply_ingest_to_memory(memory: Memory, ingest: IngestResult) -> None:
+    """Attach graph substrate metadata to a Memory record."""
+    memory.node_ids = ingest.node_ids
+    memory.edge_ids = ingest.edge_ids
+    memory.entities = ingest.entities
+    memory.confidence_score = ingest.confidence
+    memory.evidence_count = ingest.evidence_count
+    if not memory.provenance:
+        memory.provenance = memory.source or "user"