Implement pluggable memory system with SQLite driver and associated configurations

Author: venkat1701

cagent-schema.json

@@ -51,6 +51,13 @@
         "$ref": "#/definitions/RAGConfig"
       }
     },
+    "memory": {
+      "type": "object",
+      "description": "Map of memory scope configurations for pluggable memory backends",
+      "additionalProperties": {
+        "$ref": "#/definitions/MemoryConfig"
+      }
+    },
     "metadata": {
       "$ref": "#/definitions/Metadata",
       "description": "Configuration metadata"
@@ -277,6 +284,13 @@
             "type": "string"
           }
         },
+        "memory": {
+          "type": "array",
+          "description": "List of memory scopes to use for this agent",
+          "items": {
+            "type": "string"
+          }
+        },
         "hooks": {
           "$ref": "#/definitions/HooksConfig",
           "description": "Lifecycle hooks for executing shell commands at various points in the agent's execution"
@@ -1297,6 +1311,80 @@
         "strategies"
       ],
       "additionalProperties": false
+    },
+    "MemoryConfig": {
+      "type": "object",
+      "description": "Memory scope configuration for pluggable memory backends supporting long-term (RAG-style) and short-term (whiteboard) strategies",
+      "required": ["kind"],
+      "properties": {
+        "kind": {
+          "type": "string",
+          "description": "Memory backend type",
+          "enum": ["sqlite", "neo4j", "qdrant", "redis", "whiteboard"]
+        },
+        "strategy": {
+          "type": "string",
+          "description": "Memory strategy: long_term (persistent RAG-style) or short_term (ephemeral whiteboard)",
+          "enum": ["long_term", "short_term"]
+        },
+        "description": {
+          "type": "string",
+          "description": "Human-readable description of this memory scope"
+        },
+        "path": {
+          "type": "string",
+          "description": "File path for file-based backends (sqlite)"
+        },
+        "ttl": {
+          "type": "integer",
+          "description": "Time-to-live in seconds for ephemeral memory (whiteboard, redis)",
+          "minimum": 0
+        },
+        "mode": {
+          "type": "string",
+          "description": "Access mode for the memory",
+          "enum": ["read_write", "read_only", "append_only"]
+        },
+        "connection": {
+          "type": "object",
+          "description": "Connection details for remote backends",
+          "properties": {
+            "url": {
+              "type": "string",
+              "description": "Connection URL for the memory backend"
+            },
+            "database": {
+              "type": "string",
+              "description": "Database name (for backends supporting multiple databases)"
+            },
+            "collection": {
+              "type": "string",
+              "description": "Collection/table name (for vector stores)"
+            },
+            "auth": {
+              "type": "object",
+              "description": "Authentication credentials",
+              "properties": {
+                "username": {
+                  "type": "string",
+                  "description": "Username for basic authentication"
+                },
+                "password": {
+                  "type": "string",
+                  "description": "Password for basic authentication"
+                },
+                "token": {
+                  "type": "string",
+                  "description": "Token for token-based authentication"
+                }
+              },
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
     }
   }
 }

examples/memory_demo.yaml

@@ -0,0 +1,80 @@
+#!/usr/bin/env cagent run
+version: "3"
+
+metadata:
+  author: Memory System Demo
+  readme: |
+    Demonstrates the pluggable memory system with both long-term and short-term memory.
+
+    Memory strategies:
+    - **Long-term (SQLite)**: Persistent memory for user facts
+    - **Short-term (Whiteboard)**: Ephemeral shared context for multi-agent collaboration
+
+    Future backends (config-ready, implementation pending):
+    - Neo4j GraphRAG for knowledge graphs
+    - Qdrant for vector-based semantic search
+    - Redis for distributed whiteboard
+
+memory:
+  # Long-term persistent memory (current implementation)
+  user_facts:
+    kind: sqlite
+    strategy: long_term
+    path: ./memory/user_facts.db
+    description: "Persistent memory about the user"
+
+  # Short-term shared whiteboard (implementation pending)
+  # team_whiteboard:
+  #   kind: whiteboard
+  #   strategy: short_term
+  #   ttl: 3600  # 1 hour expiry
+  #   description: "Shared context for agent collaboration"
+
+  # GraphRAG with Neo4j (implementation pending)
+  # knowledge_graph:
+  #   kind: neo4j
+  #   strategy: long_term
+  #   connection:
+  #     url: bolt://localhost:7687
+  #     database: cagent
+  #     auth:
+  #       username: neo4j
+  #       password: ${NEO4J_PASSWORD}
+  #   description: "Knowledge graph for semantic relationships"
+
+agents:
+  root:
+    model: anthropic/claude-sonnet-4-5
+    description: "Assistant with long-term memory"
+    instruction: |
+      You are a helpful assistant with memory capabilities.
+      
+      Use the memory tool to remember things about the user.
+      Before responding, always check memories to personalize your responses.
+    memory:
+      - user_facts
+    toolsets:
+      - type: think
+
+  # Multi-agent example with shared whiteboard (pending whiteboard implementation)
+  # coordinator:
+  #   model: anthropic/claude-sonnet-4-5
+  #   description: "Coordinates team using shared whiteboard"
+  #   memory:
+  #     - team_whiteboard  # Shared with sub-agents
+  #     - user_facts       # Personal long-term memory
+  #   sub_agents:
+  #     - researcher
+  #     - writer
+  #
+  # researcher:
+  #   model: openai/gpt-4o
+  #   description: "Research specialist"
+  #   memory:
+  #     - team_whiteboard  # Shared whiteboard
+  #
+  # writer:
+  #   model: anthropic/claude-sonnet-4-5
+  #   description: "Writing specialist"
+  #   memory:
+  #     - team_whiteboard  # Shared whiteboard

pkg/config/latest/types.go

@@ -18,6 +18,7 @@ type Config struct {
 	Providers   map[string]ProviderConfig `json:"providers,omitempty"`
 	Models      map[string]ModelConfig    `json:"models,omitempty"`
 	RAG         map[string]RAGConfig      `json:"rag,omitempty"`
+	Memory      map[string]MemoryConfig   `json:"memory,omitempty"`
 	Metadata    Metadata                  `json:"metadata,omitempty"`
 	Permissions *PermissionsConfig        `json:"permissions,omitempty"`
 }
@@ -119,6 +120,7 @@ type AgentConfig struct {
 	SubAgents               []string          `json:"sub_agents,omitempty"`
 	Handoffs                []string          `json:"handoffs,omitempty"`
 	RAG                     []string          `json:"rag,omitempty"`
+	Memory                  []string          `json:"memory,omitempty"`
 	AddDate                 bool              `json:"add_date,omitempty"`
 	AddEnvironmentInfo      bool              `json:"add_environment_info,omitempty"`
 	CodeModeTools           bool              `json:"code_mode_tools,omitempty"`
@@ -1003,3 +1005,45 @@ func (h *HookDefinition) validate(prefix string, index int) error {
 
 	return nil
 }
+
+// MemoryConfig represents a named memory scope configuration.
+// Memory scopes define how agents store and retrieve information.
+type MemoryConfig struct {
+	// Kind specifies the memory backend type: sqlite, whiteboard, neo4j, qdrant, redis.
+	Kind string `json:"kind"`
+	// Strategy specifies the memory strategy: "long_term" (persistent RAG-style) or "short_term" (ephemeral whiteboard).
+	// Default is "long_term" for sqlite/neo4j/qdrant, "short_term" for whiteboard/redis.
+	Strategy string `json:"strategy,omitempty"`
+	// Description is an optional human-readable description of this memory scope.
+	Description string `json:"description,omitempty"`
+	// Connection holds connection details for remote backends.
+	Connection *MemoryConnectionConfig `json:"connection,omitempty"`
+	// Path is the file path for file-based backends like sqlite.
+	Path string `json:"path,omitempty"`
+	// TTL is the time-to-live in seconds for ephemeral memory (e.g., whiteboard). 0 means no expiry.
+	TTL int `json:"ttl,omitempty"`
+	// Mode specifies the access mode: "read_write" (default), "read_only", or "append_only" (event-log style).
+	Mode string `json:"mode,omitempty"`
+}
+
+// MemoryConnectionConfig holds connection details for remote memory backends.
+type MemoryConnectionConfig struct {
+	// URL is the connection URL for the memory backend.
+	URL string `json:"url"`
+	// Database is the database name (for backends that support multiple databases).
+	Database string `json:"database,omitempty"`
+	// Collection is the collection/table name (for vector stores).
+	Collection string `json:"collection,omitempty"`
+	// Auth holds authentication credentials.
+	Auth *MemoryAuthConfig `json:"auth,omitempty"`
+}
+
+// MemoryAuthConfig holds authentication credentials for memory backends.
+type MemoryAuthConfig struct {
+	// Username for basic authentication.
+	Username string `json:"username,omitempty"`
+	// Password for basic authentication.
+	Password string `json:"password,omitempty"`
+	// Token for token-based authentication.
+	Token string `json:"token,omitempty"`
+}

pkg/config/latest/validate.go

@@ -2,6 +2,7 @@ package latest
 
 import (
 	"errors"
+	"fmt"
 	"strings"
 )
 
@@ -16,8 +17,7 @@ func (t *Config) UnmarshalYAML(unmarshal func(any) error) error {
 }
 
 func (t *Config) validate() error {
-	for i := range t.Agents {
-		agent := t.Agents[i]
+	for agentName, agent := range t.Agents {
 		for j := range agent.Toolsets {
 			if err := agent.Toolsets[j].validate(); err != nil {
 				return err
@@ -28,6 +28,18 @@ func (t *Config) validate() error {
 				return err
 			}
 		}
+		// Validate agent memory references exist in top-level memory map
+		for _, memRef := range agent.Memory {
+			if _, exists := t.Memory[memRef]; !exists {
+				return fmt.Errorf("agent %q: references undefined memory %q", agentName, memRef)
+			}
+		}
+	}
+
+	for name, mem := range t.Memory {
+		if err := mem.validate(name); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -123,3 +135,89 @@ func (t *Toolset) validate() error {
 
 	return nil
 }
+
+func (m *MemoryConfig) validate(name string) error {
+	if m.Kind == "" {
+		return fmt.Errorf("memory %q: kind is required", name)
+	}
+
+	validKinds := map[string]bool{
+		"sqlite":     true,
+		"neo4j":      true,
+		"qdrant":     true,
+		"redis":      true,
+		"whiteboard": true,
+	}
+	if !validKinds[m.Kind] {
+		return fmt.Errorf("memory %q: invalid kind %q, must be one of: sqlite, neo4j, qdrant, redis, whiteboard", name, m.Kind)
+	}
+
+	// Validate strategy if provided
+	if m.Strategy != "" {
+		validStrategies := map[string]bool{
+			"long_term":  true, // Persistent RAG-style memory (sqlite, neo4j, qdrant)
+			"short_term": true, // Ephemeral whiteboard-style memory (whiteboard, redis)
+		}
+		if !validStrategies[m.Strategy] {
+			return fmt.Errorf("memory %q: invalid strategy %q, must be one of: long_term, short_term", name, m.Strategy)
+		}
+
+		// Validate strategy matches kind semantics
+		longTermKinds := map[string]bool{"sqlite": true, "neo4j": true, "qdrant": true}
+		shortTermKinds := map[string]bool{"whiteboard": true, "redis": true}
+
+		if m.Strategy == "long_term" && shortTermKinds[m.Kind] {
+			return fmt.Errorf("memory %q: kind %q is not suitable for long_term strategy (use sqlite, neo4j, or qdrant)", name, m.Kind)
+		}
+		if m.Strategy == "short_term" && longTermKinds[m.Kind] && m.Kind != "redis" {
+			// Note: redis can be used for both strategies; sqlite/neo4j/qdrant are long-term only
+			if m.Kind != "redis" {
+				return fmt.Errorf("memory %q: kind %q is not suitable for short_term strategy (use whiteboard or redis)", name, m.Kind)
+			}
+		}
+	}
+
+	// Validate mode if provided
+	if m.Mode != "" {
+		validModes := map[string]bool{
+			"read_write":  true, // Default: full read/write access
+			"read_only":   true, // Read-only access (useful for shared knowledge bases)
+			"append_only": true, // Append-only (event-log style, no updates/deletes)
+		}
+		if !validModes[m.Mode] {
+			return fmt.Errorf("memory %q: invalid mode %q, must be one of: read_write, read_only, append_only", name, m.Mode)
+		}
+	}
+
+	// Validate auth completeness if present (check Connection is not nil first)
+	if m.Connection != nil && m.Connection.Auth != nil {
+		auth := m.Connection.Auth
+		hasUserPass := auth.Username != "" || auth.Password != ""
+		hasToken := auth.Token != ""
+
+		if hasUserPass && hasToken {
+			return fmt.Errorf("memory %q: auth must use either username/password or token, not both", name)
+		}
+		if hasUserPass && (auth.Username == "" || auth.Password == "") {
+			return fmt.Errorf("memory %q: auth requires both username and password when using user/password auth", name)
+		}
+	}
+
+	// For sqlite, path is required
+	if m.Kind == "sqlite" && m.Path == "" {
+		return fmt.Errorf("memory %q: sqlite requires a path", name)
+	}
+
+	// For remote backends, connection URL is typically required
+	remoteKinds := map[string]bool{"neo4j": true, "qdrant": true, "redis": true}
+	if remoteKinds[m.Kind] && (m.Connection == nil || m.Connection.URL == "") {
+		return fmt.Errorf("memory %q: %s requires connection.url", name, m.Kind)
+	}
+
+	// TTL validation: only meaningful for short-term/ephemeral memory
+	if m.TTL > 0 && m.Kind != "whiteboard" && m.Kind != "redis" {
+		return fmt.Errorf("memory %q: ttl is only supported for whiteboard and redis kinds", name)
+	}
+
+	return nil
+}