db migration update (#679)

* adding migration from prior versions and a dump of current version database * few fixes * more fixes * moar * add pip list to pipeline * activate * fix pipeline again * dont use class info unless needed * deserialize * fix type annotation of ground_truth * update SimpleWebPageReader import * notes * missed some import changes * more fixes * type annot * more fixes * more fixes * add Feedback info instead of FeedbackDefinition info
truera · Dec 15, 2023 · 08d2b1b · 08d2b1b
1 parent 6289615
commit 08d2b1b
Show file tree

Hide file tree

Showing 32 changed files with 757 additions and 140 deletions.
diff --git a/.azure_pipelines/ci-eval.yaml b/.azure_pipelines/ci-eval.yaml
@@ -98,10 +98,16 @@ jobs:
           cd ./trulens_eval
           pip install pytest==7.0.1 pytest-azurepipelines
           pip install -r trulens_eval/requirements.optional.txt
+        displayName: Install optional deps
 
-          echo '::group::piplist'
+      - bash: |
+          source activate $(condaEnvFileSuffix)
           echo "$(pip list)"
-          echo '::endgroup::'
+        displayName: Pip list
+
+      - bash: |
+          source activate $(condaEnvFileSuffix)
+          cd ./trulens_eval
           
           python -m pytest $(testSubdirectory)
         displayName: Run notebook tests

diff --git a/docs/trulens_eval/llama_index_instrumentation.ipynb b/docs/trulens_eval/llama_index_instrumentation.ipynb
@@ -63,7 +63,7 @@
     "tru_query_engine_recorder = TruLlama(query_engine)\n",
     "\n",
     "with tru_query_engine_recorder as recording:\n",
-    "    llm_response = query_engine.query(\"What did the author do growing up?\")"
+    "    print(query_engine.query(\"What did the author do growing up?\"))"
    ]
   },
   {
@@ -142,7 +142,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from llama_index import VectorStoreIndex, SimpleWebPageReader\n",
+    "from llama_index import VectorStoreIndex\n",
+    "from llama_index.readers.web import SimpleWebPageReader\n",
     "from trulens_eval import TruLlama\n",
     "\n",
     "documents = SimpleWebPageReader(html_to_text=True).load_data(\n",
@@ -201,7 +202,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.11.6"
   },
   "orig_nbformat": 4,
   "vscode": {

diff --git a/docs/trulens_eval/logging.ipynb b/docs/trulens_eval/logging.ipynb
@@ -21,6 +21,39 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from IPython.display import JSON\n",
+    "\n",
+    "# Imports main tools:\n",
+    "from trulens_eval import Feedback\n",
+    "from trulens_eval import Huggingface\n",
+    "from trulens_eval import Tru\n",
+    "from trulens_eval import TruChain\n",
+    "from trulens_eval.schema import FeedbackResult\n",
+    "\n",
+    "tru = Tru()\n",
+    "\n",
+    "Tru().migrate_database()\n",
+    "\n",
+    "from langchain.chains import LLMChain\n",
+    "from langchain.llms import OpenAI\n",
+    "from langchain.prompts import ChatPromptTemplate\n",
+    "from langchain.prompts import HumanMessagePromptTemplate\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "\n",
+    "full_prompt = HumanMessagePromptTemplate(\n",
+    "    prompt=PromptTemplate(\n",
+    "        template=\n",
+    "        \"Provide a helpful response with relevant background information for the following: {prompt}\",\n",
+    "        input_variables=[\"prompt\"],\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "chat_prompt_template = ChatPromptTemplate.from_messages([full_prompt])\n",
+    "\n",
+    "llm = OpenAI(temperature=0.9, max_tokens=128)\n",
+    "\n",
+    "chain = LLMChain(llm=llm, prompt=chat_prompt_template, verbose=True)\n",
+    "\n",
     "truchain = TruChain(\n",
     "    chain,\n",
     "    app_id='Chain1_ChatApplication',\n",
@@ -37,6 +70,21 @@
     "Feedback functions can also be logged automatically by providing them in a list to the feedbacks arg."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize Huggingface-based feedback function collection class:\n",
+    "hugs = Huggingface()\n",
+    "\n",
+    "# Define a language match feedback function using HuggingFace.\n",
+    "f_lang_match = Feedback(hugs.language_match).on_input_output()\n",
+    "# By default this will check language match on the main app input and main app\n",
+    "# output."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -147,9 +195,11 @@
    "outputs": [],
    "source": [
     "thumb_result = True\n",
-    "tru.add_feedback(name=\"👍 (1) or 👎 (0)\", \n",
-    "                  record_id=record.record_id, \n",
-    "                  result=thumb_result)"
+    "tru.add_feedback(\n",
+    "    name=\"👍 (1) or 👎 (0)\", \n",
+    "    record_id=record.record_id, \n",
+    "    result=thumb_result\n",
+    ")"
    ]
   },
   {
@@ -177,7 +227,8 @@
     "    record=record,\n",
     "    feedback_functions=[f_lang_match]\n",
     ")\n",
-    "display(feedback_results)"
+    "for result in feedback_results:\n",
+    "    display(result)"
    ]
   },
   {
@@ -225,9 +276,11 @@
     "    feedback_mode=\"deferred\"\n",
     ")\n",
     "\n",
+    "with truchain:\n",
+    "    chain(\"This will be logged by deferred evaluator.\")\n",
+    "\n",
     "tru.start_evaluator()\n",
-    "truchain(\"This will be logged by deferred evaluator.\")\n",
-    "tru.stop_evaluator()"
+    "# tru.stop_evaluator()"
    ]
   }
  ],
@@ -247,7 +300,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.8.16"
   }
  },
  "nbformat": 4,

diff --git a/trulens_eval/DEPRECATION.md b/trulens_eval/DEPRECATION.md
@@ -3,10 +3,13 @@
 ## Changes in 0.19.0
 
 - Migrated from pydantic v1 to v2 incurring various changes.
-- `ObjSerial` class removed. `Obj` now indicate whether they are loadable when
-  `init_bindings` is not None.
 - `SingletonPerName` field `instances` renamed to `_instances` due to possible
   shadowing of `instances` field in subclassed models.
+
+### Breaking DB changes (migration script should be able to take care of these)
+
+- `ObjSerial` class removed. `Obj` now indicate whether they are loadable when
+  `init_bindings` is not None.
 - `WithClassInfo` field `__tru_class_info` renamed to `tru_class_info`
   as pydantic does not allow underscore fields.
 

diff --git a/trulens_eval/examples/experimental/dashboard_appui.ipynb b/trulens_eval/examples/experimental/dashboard_appui.ipynb
@@ -247,7 +247,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.5"
+   "version": "3.8.16"
   }
  },
  "nbformat": 4,