fix: merge EMBEDDING_MODEL_LIMITS and remove redundant validation

- Merged upstream's model list with our corrected token limits
- Kept our corrected nomic-embed-text: 2048 (not 512)
- Removed post-chunking validation (redundant with embedding-time truncation)
- All tests passing except 2 pre-existing integration test failures

Author: ww2283

packages/leann-core/src/leann/chunking_utils.py

@@ -406,13 +406,6 @@ def create_text_chunks(
 
     logger.info(f"Total chunks created: {len(all_chunks)}")
 
-    # Validate chunk token limits (default to 512 for safety)
-    # This provides a safety net for embedding models with token limits
-    validated_chunks, num_truncated = validate_chunk_token_limits(all_chunks, max_tokens=512)
-
-    if num_truncated > 0:
-        logger.info(
-            f"Post-chunking validation: {num_truncated} chunks were truncated to fit 512 token limit"
-        )
-
-    return validated_chunks
+    # Note: Token truncation is now handled at embedding time with dynamic model limits
+    # See get_model_token_limit() and truncate_to_token_limit() in embedding_compute.py
+    return all_chunks

packages/leann-core/src/leann/embedding_compute.py

@@ -109,9 +109,14 @@ def get_model_token_limit(model_name: str) -> int:
 # Ollama models use dynamic discovery via /api/show
 EMBEDDING_MODEL_LIMITS = {
     # Nomic models (common across servers)
-    "nomic-embed-text": 2048,
+    "nomic-embed-text": 2048,  # Corrected from 512 - verified via /api/show
     "nomic-embed-text-v1.5": 2048,
     "nomic-embed-text-v2": 512,
+    # Other embedding models
+    "mxbai-embed-large": 512,
+    "all-minilm": 512,
+    "bge-m3": 8192,
+    "snowflake-arctic-embed": 512,
     # OpenAI models
     "text-embedding-3-small": 8192,
     "text-embedding-3-large": 8192,
@@ -216,17 +221,6 @@ def _query_ollama_context_limit(model_name: str, base_url: str) -> Optional[int]
 # Global model cache to avoid repeated loading
 _model_cache: dict[str, Any] = {}
 
-# Known embedding model token limits
-EMBEDDING_MODEL_LIMITS = {
-    "nomic-embed-text": 512,
-    "nomic-embed-text-v2": 512,
-    "mxbai-embed-large": 512,
-    "all-minilm": 512,
-    "bge-m3": 8192,
-    "snowflake-arctic-embed": 512,
-    # Add more models as needed
-}
-
 
 def compute_embeddings(
     texts: list[str],