extend data loader

Author: ilvalerione

src/RAG/DataLoader/FileDataLoader.php

@@ -3,6 +3,7 @@
 namespace NeuronAI\RAG\DataLoader;
 
 use NeuronAI\RAG\Document;
+use NeuronAI\RAG\Splitters\DocumentSplitter;
 
 class FileDataLoader implements DataLoaderInterface
 {
@@ -79,7 +80,7 @@ protected function getDocumentsFromDirectory(string $directory): array
             closedir($handle);
         }
 
-        return $documents;
+        return DocumentSplitter::splitDocuments($documents);
     }
 
     /**

src/RAG/DataLoader/StringDataLoader.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace NeuronAI\RAG\DataLoader;
+
+use NeuronAI\RAG\Document;
+use NeuronAI\RAG\Splitters\DocumentSplitter;
+
+class StringDataLoader implements DataLoaderInterface
+{
+    public function __construct(protected string $content) {}
+
+    public function getDocuments(): array
+    {
+        return DocumentSplitter::splitDocument(new Document($this->content));
+    }
+}