Automated File Generation from Docs Notebook Changes (#1020)

Co-authored-by: joshreini1 <[email protected]> Co-authored-by: Josh Reini <[email protected]>
truera · Mar 22, 2024 · 15fe242 · 15fe242
1 parent 0387610
commit 15fe242
Show file tree

Hide file tree

Showing 20 changed files with 123 additions and 87 deletions.
diff --git a/README.md b/README.md
@@ -38,7 +38,7 @@ you to identify failure modes & systematically iterate to improve your
 application.
 
 Read more about the core concepts behind TruLens including [Feedback
-Functions](https://www.trulens.org/trulens_eval/getting_started/core_concepts/feedback_functions/),
+Functions](https://www.trulens.org/trulens_eval/getting_started/core_concepts/
 [The RAG Triad](https://www.trulens.org/trulens_eval/getting_started/core_concepts/rag_triad/),
 and [Honest, Harmless and Helpful
 Evals](https://www.trulens.org/trulens_eval/getting_started/core_concepts/honest_harmless_helpful_evals/).
@@ -94,6 +94,7 @@ guide](https://www.trulens.org/trulens_eval/contributing/) for more details.
 end of trulens_eval/gh_top_intro.md
 -->
 
+
 <!---
 start of trulens_explain/gh_top_intro.md
 NOTE: This content is from trulens_explain/gh_top_intro.md and is merged into
@@ -113,22 +114,19 @@ These installation instructions assume that you have conda installed and added
 to your path.
 
 0. Create a virtual environment (or modify an existing one).
-
 ```bash
 conda create -n "<my_name>" python=3  # Skip if using existing environment.
 conda activate <my_name>
 ```
-
+ 
 1. Install dependencies.
-
 ```bash
 conda install tensorflow-gpu=1  # Or whatever backend you're using.
 conda install keras             # Or whatever backend you're using.
 conda install matplotlib        # For visualizations.
 ```
 
 2. [Pip installation] Install the trulens pip package from PyPI.
-
 ```bash
 pip install trulens
 ```
@@ -154,10 +152,10 @@ pip install git+https://github.com/truera/trulens@BRANCH#subdirectory=trulens_ex
 To quickly play around with the TruLens library, check out the following Colab
 notebooks:
 
-- PyTorch: [![Open In
-Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1n77IGrPDO2XpeIVo_LQW0gY78enV-tY9?usp=sharing)
-- TensorFlow 2 / Keras: [![Open In
-Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1f-ETsdlppODJGQCdMXG-jmGmfyWyW2VD?usp=sharing)
+* PyTorch: [![Open In
+  Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1n77IGrPDO2XpeIVo_LQW0gY78enV-tY9?usp=sharing)
+* TensorFlow 2 / Keras: [![Open In
+  Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1f-ETsdlppODJGQCdMXG-jmGmfyWyW2VD?usp=sharing)
 
 For more information, see [TruLens-Explain
 Documentation](https://www.trulens.org/trulens_explain/getting_started/quickstart/).

diff --git a/trulens_eval/setup.py b/trulens_eval/setup.py
@@ -44,8 +44,7 @@ def run(self):
     packages=find_namespace_packages(
         include=["trulens_eval", "trulens_eval.*"]
     ),
-    python_requires=
-    '>= 3.8, < 3.13',
+    python_requires='>= 3.8, < 3.13',
     entry_points={
         'console_scripts': [
             'trulens-eval=trulens_eval.utils.command_line:main'

diff --git a/trulens_eval/trulens_eval/app.py b/trulens_eval/trulens_eval/app.py
@@ -16,9 +16,10 @@
 import queue
 import threading
 from threading import Lock
-from typing import (Any, Awaitable, Callable, ClassVar, Dict, Hashable,
-                    Iterable, List, Optional, Sequence, Set, Tuple, Type,
-                    TypeVar)
+from typing import (
+    Any, Awaitable, Callable, ClassVar, Dict, Hashable, Iterable, List,
+    Optional, Sequence, Set, Tuple, Type, TypeVar
+)
 
 import pydantic
 
@@ -459,14 +460,18 @@ class App(AppDefinition, WithInstrumentCallbacks, Hashable):
     )
     """Feedback functions to evaluate on each record."""
 
-    tru: Optional[trulens_eval.tru.Tru] = pydantic.Field(default=None, exclude=True)
+    tru: Optional[trulens_eval.tru.Tru] = pydantic.Field(
+        default=None, exclude=True
+    )
     """Workspace manager.
     
     If this is not povided, a singleton [Tru][trulens_eval.tru.Tru] will be made
     (if not already) and used.
     """
 
-    db: Optional[trulens_eval.db.DB] = pydantic.Field(default=None, exclude=True)
+    db: Optional[trulens_eval.db.DB] = pydantic.Field(
+        default=None, exclude=True
+    )
     """Database interface.
     
     If this is not provided, a singleton
@@ -655,8 +660,6 @@ def _tru_post_init(self):
         
         """
 
-
-
         if self.tru is None:
             if self.feedback_mode != FeedbackMode.NONE:
                 from trulens_eval.tru import Tru
@@ -698,14 +701,14 @@ def _tru_post_init(self):
                     raise Exception(
                         f"Feedback function {f} is not loadable. Cannot use DEFERRED feedback mode. {e}"
                     ) from e
-        
+
         if not self.selector_nocheck:
 
             dummy = self.dummy_record()
 
             for feedback in self.feedbacks:
                 feedback.check_selectors(
-                    app=self, 
+                    app=self,
                     # Don't have a record yet, but use an empty one for the non-call related fields.
                     record=dummy,
                     warning=self.selector_check_warning
@@ -1357,11 +1360,11 @@ def dummy_record(
         cost: Cost = mod_schema.Cost(),
         perf: Perf = mod_schema.Perf.now(),
         ts: datetime.datetime = datetime.datetime.now(),
-        main_input: str ="main_input are strings.",
-        main_output: str ="main_output are strings.",
+        main_input: str = "main_input are strings.",
+        main_output: str = "main_output are strings.",
         main_error: str = "main_error are strings.",
-        meta: Dict ={'metakey': 'meta are dicts'},
-        tags: str ='tags are strings'
+        meta: Dict = {'metakey': 'meta are dicts'},
+        tags: str = 'tags are strings'
     ) -> Record:
         """Create a dummy record with some of the expected structure without
         actually invoking the app.
@@ -1399,7 +1402,11 @@ def dummy_record(
                         sample_args[p.name] = p.default
 
                 sample_call = RecordAppCall(
-                    stack=[mod_schema.RecordAppCallMethod(path=lens, method=method_serial)],
+                    stack=[
+                        mod_schema.RecordAppCallMethod(
+                            path=lens, method=method_serial
+                        )
+                    ],
                     args=sample_args,
                     rets=None,
                     pid=0,
@@ -1471,6 +1478,6 @@ def print_instrumented_components(self) -> None:
 
         print("\n".join(object_strings))
 
+
 # NOTE: Cannot App.model_rebuild here due to circular imports involving tru.Tru
 # and db.DB. Will rebuild each App subclass instead.
-
diff --git a/trulens_eval/trulens_eval/database/sqlalchemy_db.py b/trulens_eval/trulens_eval/database/sqlalchemy_db.py
@@ -2,8 +2,9 @@
 from datetime import datetime
 import json
 import logging
-from typing import (Any, ClassVar, Dict, Iterable, List, Optional, Sequence,
-                    Tuple, Union)
+from typing import (
+    Any, ClassVar, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+)
 import warnings
 
 import numpy as np

diff --git a/trulens_eval/trulens_eval/db.py b/trulens_eval/trulens_eval/db.py
@@ -127,7 +127,8 @@ def insert_feedback_definition(
 
     @abc.abstractmethod
     def get_feedback_defs(
-        self, feedback_definition_id: Optional[FeedbackDefinitionID] = None
+        self,
+        feedback_definition_id: Optional[FeedbackDefinitionID] = None
     ) -> pd.DataFrame:
         """Retrieve feedback definitions from the database.
         
@@ -140,7 +141,6 @@ def get_feedback_defs(
             A dataframe with the feedback definitions.
         """
 
-
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -263,7 +263,7 @@ def versioning_decorator(func):
     trulens_eval version (or rather the db version used in the current
     trulens_eval version).
     """
-    
+
     def wrapper(self, *args, **kwargs):
         db_migration._migration_checker(db=self)
         returned_value = func(self, *args, **kwargs)
@@ -845,6 +845,5 @@ def get_feedback_count_by_status(
         limit: Optional[int] = None,
         shuffle: bool = False
     ) -> Dict[FeedbackResultStatus, int]:
-        
+
         raise NotImplementedError("This database implementation is deprecated.")
-
diff --git a/trulens_eval/trulens_eval/feedback/feedback.py b/trulens_eval/trulens_eval/feedback/feedback.py
@@ -9,8 +9,9 @@
 import logging
 from pprint import pformat
 import traceback
-from typing import (Any, Callable, Dict, Iterable, List, Optional, Tuple,
-                    TypeVar, Union)
+from typing import (
+    Any, Callable, Dict, Iterable, List, Optional, Tuple, TypeVar, Union
+)
 import warnings
 
 import munch
@@ -21,11 +22,6 @@
 from rich.markdown import Markdown
 from rich.pretty import pretty_repr
 
-# WARNING: HACK014: importing schema seems to break pydantic for unknown reason.
-# This happens even if you import it as something else.
-# from trulens_eval import schema # breaks pydantic
-# from trulens_eval import schema as tru_schema # also breaks pydantic
-
 from trulens_eval.feedback.provider.base import LLMProvider
 from trulens_eval.feedback.provider.endpoint.base import Endpoint
 from trulens_eval.schema import AppDefinition
@@ -50,6 +46,11 @@
 from trulens_eval.utils.text import UNICODE_CHECK
 from trulens_eval.utils.threading import TP
 
+# WARNING: HACK014: importing schema seems to break pydantic for unknown reason.
+# This happens even if you import it as something else.
+# from trulens_eval import schema # breaks pydantic
+# from trulens_eval import schema as tru_schema # also breaks pydantic
+
 logger = logging.getLogger(__name__)
 
 A = TypeVar("A")
@@ -450,10 +451,7 @@ def aggregate(
         if combinations is not None:
             updates['combinations'] = combinations
 
-        return Feedback.model_copy(
-            self,
-            update=updates
-        )
+        return Feedback.model_copy(self, update=updates)
 
     @staticmethod
     def of_feedback_definition(f: FeedbackDefinition):
@@ -566,7 +564,7 @@ def on(self, *args, **kwargs) -> Feedback:
                 raise ValueError(
                     f"Expected a Lens but got `{path}` of type `{class_name(type(path))}`."
                 )
-            
+
             argname = self._next_unselected_arg_name()
             new_selectors[argname] = path
             self._print_guessed_selector(argname, path)
@@ -627,10 +625,15 @@ def check_selectors(
             source_data = {}
 
         app_type: str = "trulens recorder (`TruChain`, `TruLlama`, etc)"
-    
+
         if isinstance(app, App):
             app_type = f"`{type(app).__name__}`"
-            app = jsonify(app, instrument=app.instrument, skip_specials=True, redact_keys=True)
+            app = jsonify(
+                app,
+                instrument=app.instrument,
+                skip_specials=True,
+                redact_keys=True
+            )
 
         elif isinstance(app, AppDefinition):
             app = jsonify(app, skip_specials=True, redact_keys=True)
@@ -681,14 +684,18 @@ def check_selectors(
             if prefix is None:
                 continue
 
-            if len(prefix.path) >= 2 and isinstance(prefix.path[-1], GetItemOrAttribute) and prefix.path[-1].get_item_or_attribute() == "rets":
+            if len(prefix.path) >= 2 and isinstance(
+                    prefix.path[-1], GetItemOrAttribute
+            ) and prefix.path[-1].get_item_or_attribute() == "rets":
                 # If the selector check failed because the selector was pointing
                 # to something beyond the rets of a record call, we have to
                 # ignore it as we cannot tell what will be in the rets ahead of
                 # invoking app.
                 continue
 
-            if len(prefix.path) >= 3 and isinstance(prefix.path[-2], GetItemOrAttribute) and prefix.path[-2].get_item_or_attribute() == "args":
+            if len(prefix.path) >= 3 and isinstance(
+                    prefix.path[-2], GetItemOrAttribute
+            ) and prefix.path[-2].get_item_or_attribute() == "args":
                 # Likewise if failure was because the selector was pointing to
                 # method args beyond their parameter names, we also cannot tell
                 # their contents so skip.
@@ -704,7 +711,10 @@ def check_selectors(
                         prefix_obj = prefix_obj.toDict()
 
                     msg += f"- Object of type `{class_name(type(prefix_obj))}` starting with:\n"
-                    msg += "```python\n" + retab(tab="\t  ", s=pretty_repr(prefix_obj, max_depth=2, indent_size=2)) + "\n```\n"
+                    msg += "```python\n" + retab(
+                        tab="\t  ",
+                        s=pretty_repr(prefix_obj, max_depth=2, indent_size=2)
+                    ) + "\n```\n"
 
             except Exception as e:
                 msg += f"Some non-existant object because: {pretty_repr(e)}"
@@ -719,8 +729,9 @@ def check_selectors(
             return False
 
         else:
-            raise ValueError("Some selectors do not exist in the app or record.")
-
+            raise ValueError(
+                "Some selectors do not exist in the app or record."
+            )
 
     def run(
         self,

diff --git a/trulens_eval/trulens_eval/feedback/provider/endpoint/openai.py b/trulens_eval/trulens_eval/feedback/provider/endpoint/openai.py
@@ -228,7 +228,8 @@ def __init__(
             # Already created with SingletonPerName mechanism
             if len(kwargs) != 0:
                 logger.warning(
-                    "OpenAIClient singleton already made, ignoring arguments %s", kwargs
+                    "OpenAIClient singleton already made, ignoring arguments %s",
+                    kwargs
                 )
             return
 
@@ -252,7 +253,8 @@ def __init__(
         else:
             if len(kwargs) != 0:
                 logger.warning(
-                    "Arguments %s are ignored as `client` was provided.", list(kwargs.keys())
+                    "Arguments %s are ignored as `client` was provided.",
+                    list(kwargs.keys())
                 )
 
             # Convert openai client to our wrapper if needed.

diff --git a/trulens_eval/trulens_eval/feedback/provider/litellm.py b/trulens_eval/trulens_eval/feedback/provider/litellm.py
@@ -105,12 +105,14 @@ def _create_chat_completion(
         completion_args.update(self.completion_args)
 
         if prompt is not None:
-            completion_args['messages'] = [{
+            completion_args['messages'] = [
+                {
                     "role": "system",
                     "content": prompt
-                }]
+                }
+            ]
         elif messages is not None:
-            completion_args['messages']=messages
+            completion_args['messages'] = messages
 
         else:
             raise ValueError("`prompt` or `messages` must be specified.")

diff --git a/trulens_eval/trulens_eval/feedback/provider/openai.py b/trulens_eval/trulens_eval/feedback/provider/openai.py
@@ -73,9 +73,7 @@ def __init__(
         self_kwargs['model_engine'] = model_engine
 
         self_kwargs['endpoint'] = OpenAIEndpoint(
-            *args, 
-            pace=pace, rpm=rpm,
-            **kwargs
+            *args, pace=pace, rpm=rpm, **kwargs
         )
 
         super().__init__(