diff --git a/.gitignore b/.gitignore index f72e561..77c444b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ +*.venv .DS_Store .benchmarks_venv __pycache__ /benchmarks/mlcraft +/vannaai/.files diff --git a/superset/README_RU.md b/superset/README_RU.md index a1bbdc3..e2dedbe 100644 --- a/superset/README_RU.md +++ b/superset/README_RU.md @@ -74,9 +74,7 @@ Synmetrix предоставляет начальные данные для де ## Запрос данных из Synmetrix в Superset -После установления соединения данные Synmetrix становятся доступны как таблицы в Superset. Это позволяет пользователям создавать наборы данных и проектировать диаграммы, использу - -я структурированные модели данных Synmetrix. Например, рассмотрим модель данных для "orders": +После установления соединения данные Synmetrix становятся доступны как таблицы в Superset. Это позволяет пользователям создавать наборы данных и проектировать диаграммы, используя структурированные модели данных Synmetrix. Например, рассмотрим модель данных для "orders": ```yaml cubes: diff --git a/vannaai/.chainlit/config.toml b/vannaai/.chainlit/config.toml new file mode 100644 index 0000000..6d29d95 --- /dev/null +++ b/vannaai/.chainlit/config.toml @@ -0,0 +1,120 @@ +[project] +# Whether to enable telemetry (default: true). No personal data is collected. +enable_telemetry = false + +# List of environment variables to be provided by each user to use the app. +user_env = [] + +# Duration (in seconds) during which the session is saved when the connection is lost +session_timeout = 3600 + +# Enable third parties caching (e.g LangChain cache) +cache = false + +# Authorized origins +allow_origins = ["*"] + +# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317) +# follow_symlink = false + +[features] +# Show the prompt playground +prompt_playground = true + +# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript) +unsafe_allow_html = false + +# Process and display mathematical expressions. This can clash with "$" characters in messages. +latex = false + +# Automatically tag threads with the current chat profile (if a chat profile is used) +auto_tag_thread = true + +# Authorize users to spontaneously upload files with messages +[features.spontaneous_file_upload] + enabled = false + accept = ["*/*"] + max_files = 20 + max_size_mb = 500 + +[features.audio] + # Threshold for audio recording + min_decibels = -45 + # Delay for the user to start speaking in MS + initial_silence_timeout = 3000 + # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop. + silence_timeout = 1500 + # Above this duration (MS), the recording will forcefully stop. + max_duration = 15000 + # Duration of the audio chunks in MS + chunk_duration = 1000 + # Sample rate of the audio + sample_rate = 44100 + +[UI] +# Name of the app and chatbot. +name = "Synmetrix Chatbot" + +# Show the readme while the thread is empty. +show_readme_as_default = true + +# Description of the app and chatbot. This is used for HTML tags. +# description = "" + +# Large size content are by default collapsed for a cleaner ui +default_collapse_content = true + +# The default value for the expand messages settings. +default_expand_messages = false + +# Hide the chain of thought details from the user in the UI. +hide_cot = false + +# Link to your github repo. This will add a github button in the UI's header. +github = "https://github.com/mlcraft-io/mlcraft" + +# Specify a CSS file that can be used to customize the user interface. +# The CSS file can be served from the public directory or via an external link. +# custom_css = "/public/test.css" + +# Specify a Javascript file that can be used to customize the user intejrface. +# The Javascript file can be served from the public directory. +# custom_js = "/custom.js" + +# Specify a custom font url. +# custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap" + +# Specify a custom meta image url. +# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png" + +# Specify a custom build directory for the frontend. +# This can be used to customize the frontend code. +# Be careful: If this is a relative path, it should not start with a slash. +# custom_build = "./public/build" + +[UI.theme] + #layout = "wide" + #font_family = "Inter, sans-serif" +# Override default MUI light theme. (Check theme.ts) +[UI.theme.light] + #background = "#FAFAFA" + #paper = "#FFFFFF" + + [UI.theme.light.primary] + #main = "#F80061" + #dark = "#980039" + #light = "#FFE7EB" + +# Override default MUI dark theme. (Check theme.ts) +[UI.theme.dark] + #background = "#FAFAFA" + #paper = "#FFFFFF" + + [UI.theme.dark.primary] + #main = "#F80061" + #dark = "#980039" + #light = "#FFE7EB" + + +[meta] +generated_by = "1.1.202" diff --git a/vannaai/.chainlit/translations/en-US.json b/vannaai/.chainlit/translations/en-US.json new file mode 100644 index 0000000..0bca720 --- /dev/null +++ b/vannaai/.chainlit/translations/en-US.json @@ -0,0 +1,231 @@ +{ + "components": { + "atoms": { + "buttons": { + "userButton": { + "menu": { + "settings": "Settings", + "settingsKey": "S", + "APIKeys": "API Keys", + "logout": "Logout" + } + } + } + }, + "molecules": { + "newChatButton": { + "newChat": "New Chat" + }, + "tasklist": { + "TaskList": { + "title": "\ud83d\uddd2\ufe0f Task List", + "loading": "Loading...", + "error": "An error occured" + } + }, + "attachments": { + "cancelUpload": "Cancel upload", + "removeAttachment": "Remove attachment" + }, + "newChatDialog": { + "createNewChat": "Create new chat?", + "clearChat": "This will clear the current messages and start a new chat.", + "cancel": "Cancel", + "confirm": "Confirm" + }, + "settingsModal": { + "settings": "Settings", + "expandMessages": "Expand Messages", + "hideChainOfThought": "Hide Chain of Thought", + "darkMode": "Dark Mode" + }, + "detailsButton": { + "using": "Using", + "running": "Running", + "took_one": "Took {{count}} step", + "took_other": "Took {{count}} steps" + }, + "auth": { + "authLogin": { + "title": "Login to access the app.", + "form": { + "email": "Email address", + "password": "Password", + "noAccount": "Don't have an account?", + "alreadyHaveAccount": "Already have an account?", + "signup": "Sign Up", + "signin": "Sign In", + "or": "OR", + "continue": "Continue", + "forgotPassword": "Forgot password?", + "passwordMustContain": "Your password must contain:", + "emailRequired": "email is a required field", + "passwordRequired": "password is a required field" + }, + "error": { + "default": "Unable to sign in.", + "signin": "Try signing in with a different account.", + "oauthsignin": "Try signing in with a different account.", + "redirect_uri_mismatch": "The redirect URI is not matching the oauth app configuration.", + "oauthcallbackerror": "Try signing in with a different account.", + "oauthcreateaccount": "Try signing in with a different account.", + "emailcreateaccount": "Try signing in with a different account.", + "callback": "Try signing in with a different account.", + "oauthaccountnotlinked": "To confirm your identity, sign in with the same account you used originally.", + "emailsignin": "The e-mail could not be sent.", + "emailverify": "Please verify your email, a new email has been sent.", + "credentialssignin": "Sign in failed. Check the details you provided are correct.", + "sessionrequired": "Please sign in to access this page." + } + }, + "authVerifyEmail": { + "almostThere": "You're almost there! We've sent an email to ", + "verifyEmailLink": "Please click on the link in that email to complete your signup.", + "didNotReceive": "Can't find the email?", + "resendEmail": "Resend email", + "goBack": "Go Back", + "emailSent": "Email sent successfully.", + "verifyEmail": "Verify your email address" + }, + "providerButton": { + "continue": "Continue with {{provider}}", + "signup": "Sign up with {{provider}}" + }, + "authResetPassword": { + "newPasswordRequired": "New password is a required field", + "passwordsMustMatch": "Passwords must match", + "confirmPasswordRequired": "Confirm password is a required field", + "newPassword": "New password", + "confirmPassword": "Confirm password", + "resetPassword": "Reset Password" + }, + "authForgotPassword": { + "email": "Email address", + "emailRequired": "email is a required field", + "emailSent": "Please check the email address {{email}} for instructions to reset your password.", + "enterEmail": "Enter your email address and we will send you instructions to reset your password.", + "resendEmail": "Resend email", + "continue": "Continue", + "goBack": "Go Back" + } + } + }, + "organisms": { + "chat": { + "history": { + "index": { + "showHistory": "Show history", + "lastInputs": "Last Inputs", + "noInputs": "Such empty...", + "loading": "Loading..." + } + }, + "inputBox": { + "input": { + "placeholder": "Type your message here..." + }, + "speechButton": { + "start": "Start recording", + "stop": "Stop recording" + }, + "SubmitButton": { + "sendMessage": "Send message", + "stopTask": "Stop Task" + }, + "UploadButton": { + "attachFiles": "Attach files" + }, + "waterMark": { + "text": "Built with" + } + }, + "Messages": { + "index": { + "running": "Running", + "executedSuccessfully": "executed successfully", + "failed": "failed", + "feedbackUpdated": "Feedback updated", + "updating": "Updating" + } + }, + "dropScreen": { + "dropYourFilesHere": "Drop your files here" + }, + "index": { + "failedToUpload": "Failed to upload", + "cancelledUploadOf": "Cancelled upload of", + "couldNotReachServer": "Could not reach the server", + "continuingChat": "Continuing previous chat" + }, + "settings": { + "settingsPanel": "Settings panel", + "reset": "Reset", + "cancel": "Cancel", + "confirm": "Confirm" + } + }, + "threadHistory": { + "sidebar": { + "filters": { + "FeedbackSelect": { + "feedbackAll": "Feedback: All", + "feedbackPositive": "Feedback: Positive", + "feedbackNegative": "Feedback: Negative" + }, + "SearchBar": { + "search": "Search" + } + }, + "DeleteThreadButton": { + "confirmMessage": "This will delete the thread as well as it's messages and elements.", + "cancel": "Cancel", + "confirm": "Confirm", + "deletingChat": "Deleting chat", + "chatDeleted": "Chat deleted" + }, + "index": { + "pastChats": "Past Chats" + }, + "ThreadList": { + "empty": "Empty...", + "today": "Today", + "yesterday": "Yesterday", + "previous7days": "Previous 7 days", + "previous30days": "Previous 30 days" + }, + "TriggerButton": { + "closeSidebar": "Close sidebar", + "openSidebar": "Open sidebar" + } + }, + "Thread": { + "backToChat": "Go back to chat", + "chatCreatedOn": "This chat was created on" + } + }, + "header": { + "chat": "Chat", + "readme": "Readme" + } + } + }, + "hooks": { + "useLLMProviders": { + "failedToFetchProviders": "Failed to fetch providers:" + } + }, + "pages": { + "Design": {}, + "Env": { + "savedSuccessfully": "Saved successfully", + "requiredApiKeys": "Required API Keys", + "requiredApiKeysInfo": "To use this app, the following API keys are required. The keys are stored on your device's local storage." + }, + "Page": { + "notPartOfProject": "You are not part of this project." + }, + "ResumeButton": { + "resumeChat": "Resume Chat" + } + } +} \ No newline at end of file diff --git a/vannaai/.env b/vannaai/.env new file mode 100644 index 0000000..2e2986c --- /dev/null +++ b/vannaai/.env @@ -0,0 +1,8 @@ +# set your OpenAI api key +OPENAI_API_KEY= + +PG_CONNECTION_STRING=postgres://demo_clickhouse_user:demo_clickhouse_pass@synmetrix:15432/db +QDRANT_URL=http://qdrant:6333 +CUBESTORE_VERSION=v0.35.33-arm64v8 +CUBESQL_SQL_PUSH_DOWN=true +CHAINLIT_AUTH_SECRET="yFtTf%>n6jN%~pn6MbO*aUS9CGbwp1w1vuO0h=Q10,tO4BK:HF5lK_v~GE2wu^5U" diff --git a/vannaai/1-start-containers.sh b/vannaai/1-start-containers.sh new file mode 100755 index 0000000..3ded652 --- /dev/null +++ b/vannaai/1-start-containers.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker compose pull synmetrix && docker compose up -d \ No newline at end of file diff --git a/vannaai/2-show-logs.sh b/vannaai/2-show-logs.sh new file mode 100755 index 0000000..0c4dccf --- /dev/null +++ b/vannaai/2-show-logs.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker-compose logs -f \ No newline at end of file diff --git a/vannaai/3-stop-containers.sh b/vannaai/3-stop-containers.sh new file mode 100755 index 0000000..354ca1f --- /dev/null +++ b/vannaai/3-stop-containers.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker-compose down -v \ No newline at end of file diff --git a/vannaai/Dockerfile b/vannaai/Dockerfile new file mode 100644 index 0000000..99ff3df --- /dev/null +++ b/vannaai/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.9-slim + +WORKDIR /app +RUN apt-get update && apt-get install -y gcc + +RUN pip install --upgrade pip +RUN pip install wdb + +## DEBUG python projects via WDB +ENV WDB_NO_BROWSER_AUTO_OPEN=True +# use breakpoint() +ENV PYTHONBREAKPOINT="wdb.set_trace" +ENV WDB_SOCKET_SERVER="wdb" +ENV WDB_SOCKET_PORT=19840 + +COPY requirements.txt requirements.txt + +RUN pip --no-cache-dir install -r requirements.txt + +COPY . . + +CMD ["chainlit", "run", "app.py", "-w"] diff --git a/vannaai/README.md b/vannaai/README.md new file mode 100644 index 0000000..fd92fad --- /dev/null +++ b/vannaai/README.md @@ -0,0 +1,56 @@ +# Integrating Synmetrix with Vanna+Chainlit + +[Chainlit](https://github.com/Chainlit/chainlit) — is a versatile asynchronous Python framework that enables the creation of scalable applications for conversational artificial intelligence and agent applications. Providing support for interfaces like ChatGPT, embedded chatbots, customizable user interfaces, and integration with other platforms. +[Vanna](https://github.com/vanna-ai/vanna) — is an open-source framework for generating SQL queries and related functions. Developed in Python, Vanna utilizes Retrieval-Augmented Generation (RAG) technology, which significantly simplifies the process of working with data and speeds up the retrieval of necessary information, easily converting natural language questions into precise SQL queries, making working with databases more accessible and efficient. + +### Configuring Synmetrix with Vanna+Chainlit + +Before getting started, ensure the following software is installed: + - [Docker](https://docs.docker.com/install) + - [Docker Compose](https://docs.docker.com/compose/install) + +Clone this repository: + +```bash +git clone https://github.com/mlcraft-io/examples +``` + +and navigate to the `vannaai` folder: + +```bash +cd examples/vannaai +``` + +1. Set your OpenAI API key in the OPENAI_API_KEY variable in the .env file. + +2. Use Docker Compose from this directory to start Synmetrix and Vanna+Chainlit. To get started, execute the following command: + +```bash +./1-start-containers.sh +``` + +Before proceeding, ensure you have reviewed the [Synmetrix Quick Start Guide](https://docs.synmetrix.org/docs/quickstart#step-3-explore-synmetrix). + +Wait until the services are up and running. You can check the status of services using the following command: + +```bash +./2-show-logs.sh +``` + +NOTE: To stop the services and remove volumes, execute the following command: + +```bash +./3-stop-containers.sh +``` + +### Using Synmetrix Integration with Vanna+Chainlit + +1. Go to: `http://localhost:8000` + +The integration of Synmetrix with Vanna+Chainlit allows users to retrieve data using large language models (LLMs), eliminating the need to write SQL queries. Browse generated SQL queries and charts. + +## Key References + +* [Chainlit](https://github.com/Chainlit/chainlit) +* [Vanna](https://github.com/vanna-ai/vanna) +* [SQL API](https://docs.synmetrix.org/docs/core-concepts/sql-interface) \ No newline at end of file diff --git a/vannaai/README_RU.md b/vannaai/README_RU.md new file mode 100644 index 0000000..068ea31 --- /dev/null +++ b/vannaai/README_RU.md @@ -0,0 +1,56 @@ +# Интеграция Synmetrix с Vanna+Chainlit + +[Chainlit](https://github.com/Chainlit/chainlitmlc) — это универсальный асинхронный фреймворк на Python, который позволяет создавать масштабируемые приложения для конверсационного искусственного интеллекта и агентных приложений. Обеспечивая поддержку интерфейсов вроде ChatGPT, встроенных чат-ботов, настраиваемых пользовательских интерфейсов и интеграцию с другими платформами. +[Vanna](https://github.com/vanna-ai/vanna) — это open-source фреймворк для генерации SQL-запросов и связанных с этим функций. Разработанный на языке Python, Vanna использует технологию Retrieval-Augmented Generation (RAG), что позволяет значительно упростить процесс работы с данными и ускорить получение нужной информации, легко преобразовывать вопросы на естественном языке в точные SQL-запросы, что делает работу с базами данных более доступной и эффективной. + +### Конфигурирование Synmetrix с Vanna+Chainlit + +Перед началом убедитесь, что следующее программное обеспечение установлено: + - [Docker](https://docs.docker.com/install) + - [Docker Compose](https://docs.docker.com/compose/install) + +Клонируйте этот репозиторий: + +```bash +git clone https://github.com/mlcraft-io/examples +``` + +и перейдите в папку `vannaai`: + +```bash +cd examples/vannaai +``` + +1. Установите API-ключ OpenAI в переменную OPENAI_API_KEY файла .env. + +2. Используйте Docker Compose из этого каталога для запуска Synmetrix и Vanna+Chainlit. Для начала работы выполните следующую команду: + +```bash +./1-start-containers.sh +``` + +Перед продолжением убедитесь, что вы ознакомились с [Руководством по быстрому старту Synmetrix](https://docs.synmetrix.org/docs/quickstart#step-3-explore-synmetrix). + +Дождитесь, пока сервисы не будут запущены и работают. Вы можете проверить статус сервисов с помощью следующей команды: + +```bash +./2-show-logs.sh +``` + +ПРИМЕЧАНИЕ: для остановки сервисов и удаления томов выполните следующую команду: + +```bash +./3-stop-containers.sh +``` + +### Использование интеграции Synmetrix с Vanna+Chainlit + +1. Перейдите на: `http://localhost:8000` + +Интеграция Synmetrix с Vanna+Chainlit дает пользователям возможность получать данные с помощью больших языковых моделей (LLM), устраняя необходимость написания SQL-запросов. Просматривать сгенерированне SQL-запросы и графики. + +## Стоит обратить внимание + +* [Chainlit](https://github.com/Chainlit/chainlitmlc) +* [Vanna](https://github.com/vanna-ai/vanna) +* [SQL API](https://docs.synmetrix.org/docs/core-concepts/sql-interface) \ No newline at end of file diff --git a/vannaai/app.py b/vannaai/app.py new file mode 100644 index 0000000..27bbe0a --- /dev/null +++ b/vannaai/app.py @@ -0,0 +1,147 @@ +import os +from urllib.parse import urlparse +from typing import Optional +import pandas as pd + +import chainlit as cl +import chainlit.data as cl_data +from chainlit.data.sql_alchemy import SQLAlchemyDataLayer +from vanna.openai import OpenAI_Chat +from vanna.qdrant import Qdrant_VectorStore +from vanna.exceptions import ValidationError + + +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") +PG_CONNECTION_STRING = os.getenv("PG_CONNECTION_STRING") +CHAINLIT_PG_CONNECTION_STRING = os.getenv( + "CHAINLIT_PG_CONNECTION_STRING", + "postgresql+asyncpg://chainlit:pg_pass@chainlit-postgres:5432/chainlit", +) +QDRANT_URL = os.getenv("QDRANT_URL") + +cl_data._data_layer = SQLAlchemyDataLayer(conninfo=CHAINLIT_PG_CONNECTION_STRING) + + +class Vanna(Qdrant_VectorStore, OpenAI_Chat): + def __init__(self, config): + Qdrant_VectorStore.__init__(self, config=config) + OpenAI_Chat.__init__(self, config=config) + + +vn = Vanna( + { + "url": QDRANT_URL, + "api_key": OPENAI_API_KEY, + "model": "gpt-4o", + } +) + +url = urlparse(PG_CONNECTION_STRING) +vn.connect_to_postgres( + host=url.hostname, + port=url.port, + user=url.username, + password=url.password, + dbname=url.path[1:], # Strip the leading '/' from the path to get the database name +) + +df_information_schema = vn.run_sql("SELECT * FROM INFORMATION_SCHEMA.COLUMNS") + +plan = vn.get_training_plan_generic(df_information_schema) + +vn.train(plan=plan) + + +ASSISTANT_NAME = "Synmetrix" + + +def gen_query_recursive(human_query: str, trials: int = 3): + sql_query = vn.generate_sql(human_query) + + if trials == 0: + raise ValueError("Query is not a valid SQL query") + + if not vn.is_sql_valid(sql_query): + sql_query = gen_query_recursive(human_query, trials - 1) + + return sql_query + + +@cl.step(name="Generate SQL", type="tool", language="sql", show_input=False) +async def gen_query(human_query: str): + sql_query = gen_query_recursive(human_query) + return sql_query + + +@cl.step(name="Preview dataframe", type="tool", show_input=False) +async def preview_df(df: pd.DataFrame): + return df.to_markdown(index=False) + + +@cl.step(name="Generate plotly code", type="tool", language="python", show_input=False) +async def generate_plotly_code(human_query, sql, df): + return vn.generate_plotly_code(question=human_query, sql=sql, df=df) + + +@cl.step(name="Plot", show_input=False) +async def plot(human_query, sql, df): + current_step = cl.context.current_step + plotly_code = await generate_plotly_code(human_query, sql, df) + fig = vn.get_plotly_figure(plotly_code=plotly_code, df=df) + + current_step.output = plotly_code + return fig + + +@cl.step(type="run", name=ASSISTANT_NAME) +async def chain(human_query: str): + if vn.is_sql_valid(human_query): + sql_query = human_query + else: + sql_query = await gen_query(human_query) + + try: + df = vn.run_sql(sql_query) + await preview_df(df) + + fig = await plot(human_query, sql_query, df) + elements = [cl.Plotly(name="chart", figure=fig, display="inline")] + + await cl.Message( + content=human_query, elements=elements, author=ASSISTANT_NAME + ).send() + except ValueError as e: + await cl.Message(content=f"Query failed: {e}", author=ASSISTANT_NAME).send() + except ValidationError as e: + await cl.Message(content=f"Query failed: {e}", author=ASSISTANT_NAME).send() + + +@cl.on_message +async def main(message: cl.Message): + await chain(message.content) + + +@cl.password_auth_callback +def auth_callback(username: str, password: str) -> Optional[cl.User]: + if (username, password) == ("admin", "admin"): + return cl.User(identifier="admin") + else: + return None + + +@cl.set_chat_profiles +async def chat_profile(): + return [ + cl.ChatProfile( + name=" ", + icon="/public/avatars/default.png", + markdown_description="The underlying LLM model is **GPT-4o**", + starters=[ + cl.Starter( + label="Show all available cubes", + message="Show all available tables in the database", + icon="/public/starter-icon.png", + ), + ], + ) + ] diff --git a/vannaai/chainlit.md b/vannaai/chainlit.md new file mode 100644 index 0000000..b87603e --- /dev/null +++ b/vannaai/chainlit.md @@ -0,0 +1,16 @@ +# Welcome to [Synmetrix](https://github.com/mlcraft-io/mlcraft)! + +We are excited to introduce our new integration of [Chainlit](https://github.com/Chainlit/chainlit) and [Vanna](https://github.com/vanna-ai/vanna), designed to enhance your data analytics experience. This integration allows you to interact with your data effortlessly, gaining instant insights and automating complex analytical tasks. + +## Here's what you can expect: +- **Natural Language Queries**: Retrieve data using Large Language Models (LLMs) in plain English, eliminating the need to write SQL queries. +- **Time Savings**: Automate routine data analysis tasks, giving you more time to make strategic decisions. +- **Ease of Use**: An intuitive interface and powerful features from Vanna enable you to focus on what matters most. + +> Please note that this is just an example of what you can achieve with this integration. + +If you have any questions or feedback, feel free to reach out to our support team. We are always here to assist you. + +Start exploring your data with Synmetrix today! + +Current setup is using demo credentials from Synmetrix. \ No newline at end of file diff --git a/vannaai/create-venv.sh b/vannaai/create-venv.sh new file mode 100755 index 0000000..fc6257d --- /dev/null +++ b/vannaai/create-venv.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +function create_venv { + if [ ! -d ".venv" ]; then + python3.9 -m venv .venv + fi + source .venv/bin/activate + echo "Virtual environment activated!" +} + +create_venv + +python -m pip install --upgrade pip +pip install -r requirements.txt \ No newline at end of file diff --git a/vannaai/docker-compose.yml b/vannaai/docker-compose.yml new file mode 100644 index 0000000..6141e8e --- /dev/null +++ b/vannaai/docker-compose.yml @@ -0,0 +1,85 @@ +version: '3.8' + +services: + redis: + image: redis:7.0.0 + + chainlit-postgres: + image: postgres:${POSTGRES_VERSION:-12} + volumes: + - chainlit_pg_db_data:/var/lib/postgresql/data + - ./etc/chainlit-pg-dump.sql:/docker-entrypoint-initdb.d/chainlit-pg-dump.sql + environment: + POSTGRES_USER: chainlit + POSTGRES_PASSWORD: pg_pass + POSTGRES_DB: chainlit + + postgres: + image: postgres:${POSTGRES_VERSION:-12} + volumes: + - pg_db_data:/var/lib/postgresql/data + environment: + POSTGRES_USER: synmetrix_stack + POSTGRES_PASSWORD: pg_pass + POSTGRES_DB: synmetrix_stack + + cubestore: + image: cubejs/cubestore:${CUBESTORE_VERSION:-v0.34.45} + environment: + - CUBESTORE_REMOTE_DIR=/cube/data + volumes: + - cube_store_data:/cube/data + + synmetrix: + image: synmetrix/stack:${STACK_VERSION:-latest} + restart: always + ports: + - 80:8888 # nginx app + - 4000:4000 # cubejs app + - 15432:15432 # pg api + - 13306:13306 # mysql api + environment: + # postgres database used by Hasura as a data source and to store Hasura metadata + HASURA_GRAPHQL_DATABASE_URL: postgres://synmetrix_stack:pg_pass@postgres:5432/synmetrix_stack + # secure your Hasura console via admin secret + HASURA_GRAPHQL_ADMIN_SECRET: adminsecret + # SMTP credentials to send emails from synmetrix + SMTP_HOST: ${SECRETS_SMTP_HOST} + SMTP_PORT: ${SECRETS_SMTP_PORT} + SMTP_SECURE: ${SECRETS_SMTP_SECURE} + SMTP_USER: ${SECRETS_SMTP_USER} + SMTP_PASS: ${SECRETS_SMTP_PASS} + SMTP_SENDER: ${SECRETS_SMTP_SENDER} + # S3 credentials + AWS_S3_ACCESS_KEY_ID: ${SECRETS_AWS_S3_ACCESS_KEY_ID} + AWS_S3_SECRET_ACCESS_KEY: ${SECRETS_AWS_S3_SECRET_ACCESS_KEY} + AWS_S3_BUCKET_NAME: ${SECRETS_AWS_S3_BUCKET_NAME} + + qdrant: + image: qdrant/qdrant:latest + restart: always + env_file: + - .env + volumes: + - qdrant-data:/qdrant/storage + + vanna: + restart: always + build: . + ports: + - "8000:8000" + volumes: + - .:/app + env_file: + - .env + + wdb: + image: shepilovvladislav/wdb:latest + ports: + - 1990:1984 + +volumes: + pg_db_data: + chainlit_pg_db_data: + cube_store_data: + qdrant-data: diff --git a/vannaai/etc/chainlit-pg-dump.sql b/vannaai/etc/chainlit-pg-dump.sql new file mode 100644 index 0000000..a9e4c5c --- /dev/null +++ b/vannaai/etc/chainlit-pg-dump.sql @@ -0,0 +1,67 @@ +CREATE EXTENSION pgcrypto; + +CREATE TABLE users ( + "id" UUID PRIMARY KEY, + "identifier" TEXT NOT NULL UNIQUE, + "metadata" JSONB NOT NULL, + "createdAt" TEXT +); + +INSERT INTO users ("id", "identifier", "metadata", "createdAt") VALUES (gen_random_uuid(), 'admin', '{}'::jsonb, now()); + +CREATE TABLE IF NOT EXISTS threads ( + "id" UUID PRIMARY KEY, + "createdAt" TEXT, + "name" TEXT, + "userId" UUID, + "userIdentifier" TEXT, + "tags" TEXT[], + "metadata" JSONB, + FOREIGN KEY ("userId") REFERENCES users("id") ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS steps ( + "id" UUID PRIMARY KEY, + "name" TEXT NOT NULL, + "type" TEXT NOT NULL, + "threadId" UUID NOT NULL, + "parentId" UUID, + "disableFeedback" BOOLEAN NOT NULL, + "streaming" BOOLEAN NOT NULL, + "waitForAnswer" BOOLEAN, + "isError" BOOLEAN, + "metadata" JSONB, + "tags" TEXT[], + "input" TEXT, + "output" TEXT, + "createdAt" TEXT, + "start" TEXT, + "end" TEXT, + "generation" JSONB, + "showInput" TEXT, + "language" TEXT, + "indent" INT +); + +CREATE TABLE IF NOT EXISTS elements ( + "id" UUID PRIMARY KEY, + "threadId" UUID, + "type" TEXT, + "url" TEXT, + "chainlitKey" TEXT, + "name" TEXT NOT NULL, + "display" TEXT, + "objectKey" TEXT, + "size" TEXT, + "page" INT, + "language" TEXT, + "forId" UUID, + "mime" TEXT +); + +CREATE TABLE IF NOT EXISTS feedbacks ( + "id" UUID PRIMARY KEY, + "forId" UUID NOT NULL, + "value" INT NOT NULL, + "comment" TEXT +); diff --git a/vannaai/public/avatars/default.png b/vannaai/public/avatars/default.png new file mode 100644 index 0000000..ae02781 Binary files /dev/null and b/vannaai/public/avatars/default.png differ diff --git a/vannaai/public/logo_dark.png b/vannaai/public/logo_dark.png new file mode 100644 index 0000000..68210de Binary files /dev/null and b/vannaai/public/logo_dark.png differ diff --git a/vannaai/public/logo_light.png b/vannaai/public/logo_light.png new file mode 100644 index 0000000..149a9a8 Binary files /dev/null and b/vannaai/public/logo_light.png differ diff --git a/vannaai/public/starter-icon.png b/vannaai/public/starter-icon.png new file mode 100644 index 0000000..36a183c Binary files /dev/null and b/vannaai/public/starter-icon.png differ diff --git a/vannaai/requirements.txt b/vannaai/requirements.txt new file mode 100644 index 0000000..21cc84f --- /dev/null +++ b/vannaai/requirements.txt @@ -0,0 +1,6 @@ +chainlit==1.1.301 +vanna[qdrant,openai,postgres]==0.5.5 +urllib3==2.2.1 +asyncpg==0.29.0 +SQLAlchemy==2.0.30 +aiohttp==3.9.5 \ No newline at end of file