Merge pull request #7 from WillACosta/feature/improve-ai-flow

[AUTOMATED-PR] - main <- feature/improve-ai-flow

Author: WillACosta

.DS_Store

@@ -1,19 +1,19 @@
 PORT=3000
-IS_DOCKER=false
 
 LANGCHAIN_API_KEY=
 LANGCHAIN_TRACING_V2=true
 LANGCHAIN_CALLBACKS_BACKGROUND=true
 
 LLM_API_KEY=
 
-REDIS_URL=redis://127.0.0.1:6379
-REDIS_URL_DOCKER=redis://redis:6379
+REDIS_URL=redis://redis:6379
 REDIS_USER=
 REDIS_PASSWORD=
 
-DATABASE_URL=postgresql://genaiapp:password@localhost:5432/app_db?schema=public
-DATABASE_URL_DOCKER=postgresql://genaiapp:password@postgres:5432/app_db?schema=public
+DATABASE_URL=postgresql://genaiapp:<password>@postgres:5432/app_db?schema=public
 
 JWT_SECRET=
 JWT_EXPIRATION=1h
+
+QDRANT_URL=http://qdrant:6333
+QDRANT_COLLECTION=genai_documents

.env-example

@@ -1 +1,3 @@
-## Changes made in this PR
+### Description
+
+<!-- Add a full description of the changes here -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,12 +1,13 @@
 {
   "cSpell.words": [
-    "dataprovider",
-    "dataproviders",
-    "gemini",
-    "nodenext",
-    "openapi",
-    "predev",
-    "prestart",
-    "usecases"
-  ]
+		"dataprovider",
+		"dataproviders",
+		"gemini",
+		"nodenext",
+		"openapi",
+		"predev",
+		"prestart",
+		"qdrant",
+		"usecases"
+	]
 }

.vscode/settings.json

@@ -0,0 +1,21 @@
+help:
+	@echo "Available commands:"
+	@echo "  run        Start app and services using Docker."
+	@echo "  stop-all   Stop all services running on Docker."
+	@echo "  remove     Remove all Docker containers."
+	@echo "  clean      Remove temporary and cache files."
+
+run:
+	docker compose up -d
+
+build:
+	docker compose up --build -d
+
+stop-all:
+	docker compose stop $(docker ps -q)
+
+remove:
+	docker compose down
+
+clean:
+	rm -rf node_modules/ uploads/*.pdf

Makefile

@@ -41,14 +41,18 @@ modules: application features
       utils: class and constant utilities for this module
 ```
 
-### Search In Document Flow
+### Search In Documents Flow
 
-The search in document endpoint is the most complex of this application, it uses RAG concepts to break down the provided
-PDF document into small chunks and use it as context to the follow-up questions. Also, it uses Redis to store
-and retrieve chat history during user's session.
+To perform a search in documents and ask questions about them, we have two endpoints for that:
 
-> A possible improvement, is to have a separated endpoint for uploading documents and another one for handling questions
-> to it. For this initial version the endpoint accepts one document at a time and a question is needed, check this process on the diagram below:
+- `POST /resources/docs`
+- `POST /genai/search-in-documents`
+
+On `/resources/docs` the documents are uploaded to local system and then broke down into small chunks to be stored on a vector database.
+
+The `/genai/search-in-documents` endpoint uses RAG concept, it's responsible to create an Embedding of the user question, and retrieve the most relevant chunks from vector database using it as context for follow-up questions. Also, it uses Redis to store and retrieve chat history during user's session.
+
+Check this process on the diagram below:
 
 ![GenAI Search in Document Flow](docs/genai-flow.png 'GenAI Search in Document Flow')
 
@@ -80,10 +84,10 @@ cp .env.example .env
 > From the app's root directory, run the following command to build and running docker containers:
 
 ```shell
-docker compose up --build
+make run
 ```
 
-> The application will be available at `http://localhost:3000`.<br>
+> The application will be available at `http://localhost:3000`.<br>For more commands see `Makefile` or run `make help`.
 
 ## Documentation
 
@@ -126,3 +130,5 @@ Current supported roles are: [`admin`, `user`]:
 | GET status/                    | [x]   | [x]  |
 | POST /genai/translate          | [x]   | [x]  |
 | POST /genai/search-in-document | [x]   | [x]  |
+| GET /genai/chat-history        | [x]   | [x]  |
+| POST /resources/docs           | [x]   | [x]  |

README.md

@@ -8,17 +8,14 @@ services:
       - '3000:3000'
     env_file:
       - .env
-    environment:
-      IS_DOCKER: true
-      DATABASE_URL: ${DATABASE_URL_DOCKER}
-      REDIS_URL: ${REDIS_URL_DOCKER}
     volumes:
       - ./src:/usr/workspace/app/src
       - pnpm_store:/pnpm/.pnpm-store
     command: sh -c "pnpm migrate:dev && pnpm dev"
     depends_on:
       - postgres
       - redis
+      - qdrant
 
   postgres:
     container_name: genai_app_postgres
@@ -42,7 +39,16 @@ services:
     volumes:
       - 'genai_app_redis_data:/bitnami/redis/data'
 
+  qdrant:
+    container_name: genai_app_vector_db
+    image: qdrant/qdrant
+    ports:
+      - '6333:6333'
+    volumes:
+      - 'genai_app_qdrant_data:/qdrant/storage'
+
 volumes:
   pnpm_store:
   genai_app_redis_data:
   genai_app_postgres_data:
+  genai_app_qdrant_data:

docker-compose.yaml

@@ -5,7 +5,6 @@
 	"scripts": {
 		"prestart": "node dynamic-url-config.js",
 		"migrate:dev": "npx prisma migrate dev",
-		"predev": "node dynamic-url-config.js",
 		"start": "tsx src/app.ts",
 		"dev": "tsx watch src/app.ts",
 		"postinstall": "npx prisma generate"
@@ -23,8 +22,10 @@
 		"@langchain/community": "^0.3.1",
 		"@langchain/core": "^0.3.3",
 		"@langchain/google-genai": "^0.1.0",
+		"@langchain/qdrant": "^0.1.1",
 		"@langchain/redis": "^0.1.0",
 		"@prisma/client": "5.20.0",
+		"@qdrant/js-client-rest": "^1.12.0",
 		"@types/redis": "^4.0.11",
 		"express": "^4.21.0",
 		"jsonwebtoken": "^9.0.2",