From 56ee9b20a02b23edbcffeb1f7bf0fb00dd4f4329 Mon Sep 17 00:00:00 2001 From: Josh Reini <60949774+joshreini1@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:51:07 -0500 Subject: [PATCH] Josh/langchain prompt import, tag quickstart versions (#602) * remove extra reset cell * fix langchain prompt import, quickstart * async fix imports, update install pins * fix langchain prompttemplate imports * ada embeddings in quickstart, pinned install versions * pin package versions * clear output --- .../langchain/langchain_async.ipynb | 17 +--- .../langchain/langchain_groundtruth.ipynb | 12 +-- .../langchain_model_comparison.ipynb | 4 +- .../langchain/langchain_retrieval_agent.ipynb | 21 +--- .../quickstart/groundtruth_evals.ipynb | 9 ++ .../examples/quickstart/human_feedback.ipynb | 97 ++++--------------- .../quickstart/langchain_quickstart.ipynb | 13 ++- .../examples/quickstart/prototype_evals.ipynb | 9 ++ .../examples/quickstart/quickstart.ipynb | 69 +++++++++---- .../quickstart/text2text_quickstart.ipynb | 9 ++ 10 files changed, 122 insertions(+), 138 deletions(-) diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_async.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_async.ipynb index cb06929c4..4fedb394b 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_async.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_async.ipynb @@ -28,7 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "# ! pip install trulens_eval==0.18.0 langchain>=0.0.335" + "# ! pip install trulens_eval==0.18.1 langchain>=0.0.342" ] }, { @@ -40,7 +40,7 @@ "import asyncio\n", "\n", "from langchain import LLMChain\n", - "from langchain import PromptTemplate\n", + "from langchain.prompts import PromptTemplate\n", "from langchain.callbacks import AsyncIteratorCallbackHandler\n", "from langchain.chains import LLMChain\n", "from langchain.chat_models.openai import ChatOpenAI\n", @@ -70,8 +70,8 @@ "outputs": [], "source": [ "import os\n", - "os.environ[\"HUGGINGFACE_API_KEY\"] = \"...\"\n", - "os.environ[\"OPENAI_API_KEY\"] = \"...\"" + "os.environ[\"HUGGINGFACE_API_KEY\"] = \"hf_...\"\n", + "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"" ] }, { @@ -223,13 +223,6 @@ "await task\n", "record = recording.get()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -248,7 +241,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.16" + "version": "3.11.5" }, "orig_nbformat": 4, "vscode": { diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_groundtruth.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_groundtruth.ipynb index 08126d243..6e898c958 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_groundtruth.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_groundtruth.ipynb @@ -28,7 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "# ! pip install trulens==0.11.0 langchain==0.0.283" + "# ! pip install trulens_eval==0.18.1 langchain>=0.0.342" ] }, { @@ -39,8 +39,8 @@ "source": [ "from langchain.chains import LLMChain\n", "from langchain.llms import OpenAI\n", - "from langchain.prompts.chat import ChatPromptTemplate, PromptTemplate\n", - "from langchain.prompts.chat import HumanMessagePromptTemplate\n", + "from langchain.prompts import ChatPromptTemplate, PromptTemplate\n", + "from langchain.prompts import HumanMessagePromptTemplate\n", "\n", "from trulens_eval import Feedback, Tru, TruChain\n", "from trulens_eval.feedback import GroundTruthAgreement, Huggingface\n", @@ -63,8 +63,8 @@ "outputs": [], "source": [ "import os\n", - "os.environ[\"HUGGINGFACE_API_KEY\"] = \"...\"\n", - "os.environ[\"OPENAI_API_KEY\"] = \"...\"" + "os.environ[\"HUGGINGFACE_API_KEY\"] = \"hf_...\"\n", + "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"" ] }, { @@ -189,7 +189,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.11.5" }, "vscode": { "interpreter": { diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_model_comparison.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_model_comparison.ipynb index a94dd8c9d..89d975d7c 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_model_comparison.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_model_comparison.ipynb @@ -48,8 +48,8 @@ "# ! pip install langchain>=0.0.170\n", "from langchain.chains import LLMChain\n", "from langchain.llms import OpenAI\n", - "from langchain.prompts.chat import ChatPromptTemplate, PromptTemplate\n", - "from langchain.prompts.chat import HumanMessagePromptTemplate\n", + "from langchain.prompts import ChatPromptTemplate, PromptTemplate\n", + "from langchain.prompts import HumanMessagePromptTemplate\n", "from langchain import PromptTemplate\n", "from langchain.llms import OpenAI\n", "from langchain import LLMChain\n", diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_retrieval_agent.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_retrieval_agent.ipynb index 7b0862293..ab587d023 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_retrieval_agent.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_retrieval_agent.ipynb @@ -12,11 +12,6 @@ "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/langchain_retrieval_agent.ipynb)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, { "cell_type": "code", "execution_count": null, @@ -55,7 +50,7 @@ "from trulens_eval import TruChain, Feedback, Tru, feedback, Select, FeedbackMode\n", "from trulens_eval.feedback import OpenAI as fOpenAI\n", "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n", + "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n", "tru = Tru()\n" ] }, @@ -337,20 +332,6 @@ "source": [ "tru.start_evaluator()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/trulens_eval/examples/quickstart/groundtruth_evals.ipynb b/trulens_eval/examples/quickstart/groundtruth_evals.ipynb index 1fd0f1ba7..3ed48b981 100644 --- a/trulens_eval/examples/quickstart/groundtruth_evals.ipynb +++ b/trulens_eval/examples/quickstart/groundtruth_evals.ipynb @@ -23,6 +23,15 @@ "For this quickstart, you will need Open AI keys." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ! pip install trulens==0.18.1 openai==1.3.1" + ] + }, { "cell_type": "code", "execution_count": 2, diff --git a/trulens_eval/examples/quickstart/human_feedback.ipynb b/trulens_eval/examples/quickstart/human_feedback.ipynb index b592a8318..9b1bf6615 100644 --- a/trulens_eval/examples/quickstart/human_feedback.ipynb +++ b/trulens_eval/examples/quickstart/human_feedback.ipynb @@ -11,6 +11,15 @@ "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/human_feedback.ipynb)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ! pip install trulens==0.18.1 openai==1.3.1" + ] + }, { "cell_type": "code", "execution_count": null, @@ -38,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -56,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -97,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -114,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -133,25 +142,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4addc21b832d4bebb2124f902d9f7ac0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(Button(description='👍', style=ButtonStyle()), Button(description='👎', style=ButtonStyle())))" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from ipywidgets import Button, HBox, VBox\n", "\n", @@ -176,7 +169,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -198,63 +191,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Human Feedacklatencytotal_cost
app_id
LLM App v11.01.00.000159
\n", - "
" - ], - "text/plain": [ - " Human Feedack latency total_cost\n", - "app_id \n", - "LLM App v1 1.0 1.0 0.000159" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "tru.get_leaderboard(app_ids=[tru_app.app_id])" ] diff --git a/trulens_eval/examples/quickstart/langchain_quickstart.ipynb b/trulens_eval/examples/quickstart/langchain_quickstart.ipynb index 00f12f912..0119a146c 100644 --- a/trulens_eval/examples/quickstart/langchain_quickstart.ipynb +++ b/trulens_eval/examples/quickstart/langchain_quickstart.ipynb @@ -22,6 +22,15 @@ "For this quickstart you will need Open AI and Huggingface keys" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ! pip install trulens==0.18.1 openai==1.3.1" + ] + }, { "cell_type": "code", "execution_count": null, @@ -59,8 +68,8 @@ "# ! pip install langchain>=0.0.170\n", "from langchain.chains import LLMChain\n", "from langchain.llms import OpenAI\n", - "from langchain.prompts.chat import ChatPromptTemplate, PromptTemplate\n", - "from langchain.prompts.chat import HumanMessagePromptTemplate" + "from langchain.prompts import ChatPromptTemplate, PromptTemplate\n", + "from langchain.prompts import HumanMessagePromptTemplate" ] }, { diff --git a/trulens_eval/examples/quickstart/prototype_evals.ipynb b/trulens_eval/examples/quickstart/prototype_evals.ipynb index f65af7845..bc6b4423b 100644 --- a/trulens_eval/examples/quickstart/prototype_evals.ipynb +++ b/trulens_eval/examples/quickstart/prototype_evals.ipynb @@ -22,6 +22,15 @@ "## Import libraries" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ! pip install trulens==0.18.1" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/trulens_eval/examples/quickstart/quickstart.ipynb b/trulens_eval/examples/quickstart/quickstart.ipynb index e2245b3be..8aceff119 100644 --- a/trulens_eval/examples/quickstart/quickstart.ipynb +++ b/trulens_eval/examples/quickstart/quickstart.ipynb @@ -13,6 +13,15 @@ "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/quickstart.ipynb)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ! pip install trulens==0.18.1 chromadb==0.4.18 openai==1.3.1" + ] + }, { "cell_type": "code", "execution_count": null, @@ -20,7 +29,18 @@ "outputs": [], "source": [ "import os\n", - "os.environ[\"OPENAI_API_KEY\"] = \"...\"" + "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "\n", + "oai_client = OpenAI()" ] }, { @@ -56,6 +76,18 @@ "Create a chromadb vector store in memory." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "oai_client.embeddings.create(\n", + " model=\"text-embedding-ada-002\",\n", + " input=university_info\n", + " )" + ] + }, { "cell_type": "code", "execution_count": null, @@ -63,19 +95,17 @@ "outputs": [], "source": [ "import chromadb\n", - "from chromadb.utils import embedding_functions\n", - "default_ef = embedding_functions.DefaultEmbeddingFunction()\n", - "students_embeddings = default_ef([university_info])\n", + "from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction\n", + "from openai import OpenAI\n", + "oai_client = OpenAI()\n", "\n", - "client = chromadb.Client()\n", - "vector_store = client.create_collection(name=\"Students\")\n", + "embedding_function = OpenAIEmbeddingFunction(api_key=os.environ.get('OPENAI_API_KEY'),\n", + " model_name=\"text-embedding-ada-002\")\n", "\n", - "vector_store.add(\n", - " embeddings = students_embeddings,\n", - " documents = [university_info],\n", - " metadatas = [{'source':'university info'}],\n", - " ids = [\"id1\"]\n", - ")" + "\n", + "chroma_client = chromadb.PersistentClient(path=\"./chromadb\")\n", + "vector_store = chroma_client.get_or_create_collection(name=\"Universities\",\n", + " embedding_function=embedding_function)" ] }, { @@ -95,11 +125,7 @@ "source": [ "from trulens_eval import Tru\n", "from trulens_eval.tru_custom_app import instrument\n", - "tru = Tru()\n", - "\n", - "from openai import OpenAI\n", - "\n", - "oai_client = OpenAI()" + "tru = Tru()" ] }, { @@ -239,6 +265,15 @@ " rag.query(\"When was the University of Washington founded?\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tru.get_leaderboard(app_ids=[\"RAG v1\"])" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/trulens_eval/examples/quickstart/text2text_quickstart.ipynb b/trulens_eval/examples/quickstart/text2text_quickstart.ipynb index ecbff8a35..356bca46f 100644 --- a/trulens_eval/examples/quickstart/text2text_quickstart.ipynb +++ b/trulens_eval/examples/quickstart/text2text_quickstart.ipynb @@ -22,6 +22,15 @@ "For this quickstart you will need an OpenAI Key." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ! pip install trulens==0.18.1 openai==1.3.1" + ] + }, { "cell_type": "code", "execution_count": null,