diff --git a/docs/trulens_eval/all_tools.ipynb b/docs/trulens_eval/all_tools.ipynb index 11cb631c0..7eed27fb2 100644 --- a/docs/trulens_eval/all_tools.ipynb +++ b/docs/trulens_eval/all_tools.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 📓 Langchain Quickstart\n", + "# 📓 _LangChain_ Quickstart\n", "\n", "In this quickstart you will create a simple LLM Chain and learn how to log it and get feedback on an LLM response.\n", "\n", @@ -60,7 +60,7 @@ "tru = Tru()\n", "tru.reset_database()\n", "\n", - "# Imports from langchain to build app\n", + "# Imports from LangChain to build app\n", "import bs4\n", "from langchain import hub\n", "from langchain.chat_models import ChatOpenAI\n", @@ -429,7 +429,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 📓 Llama-Index Quickstart\n", + "# 📓 _LlamaIndex_ Quickstart\n", "\n", "In this quickstart you will create a simple Llama Index app and learn how to log it and get feedback on an LLM response.\n", "\n", diff --git a/docs/trulens_eval/api/app/trurails.md b/docs/trulens_eval/api/app/trurails.md index 16856976e..543e9427b 100644 --- a/docs/trulens_eval/api/app/trurails.md +++ b/docs/trulens_eval/api/app/trurails.md @@ -1,7 +1,4 @@ -# Tru Rails (NEMO Guardrails) - -!!! Warning - This recorder is experimental. +# Tru Rails for _NeMo Guardrails_ ::: trulens_eval.tru_rails.TruRails diff --git a/docs/trulens_eval/api/provider/langchain.md b/docs/trulens_eval/api/provider/langchain.md index ef8dc3c17..b0432b90d 100644 --- a/docs/trulens_eval/api/provider/langchain.md +++ b/docs/trulens_eval/api/provider/langchain.md @@ -1,12 +1,12 @@ -# 🦜️🔗 Langchain Provider +# 🦜️🔗 _LangChain_ Provider -Below is how you can instantiate a [Langchain LLM](https://python.langchain.com/docs/modules/model_io/llms/) as a provider. +Below is how you can instantiate a [_LangChain_ LLM](https://python.langchain.com/docs/modules/model_io/llms/) as a provider. All feedback functions listed in the base [LLMProvider -class][trulens_eval.feedback.provider.base.LLMProvider] can be run with the Langchain Provider. +class][trulens_eval.feedback.provider.base.LLMProvider] can be run with the _LangChain_ Provider. !!! note - Langchain provider cannot be used in `deferred` mode due to inconsistent serialization capabilities of langchain apps. + _LangChain_ provider cannot be used in `deferred` mode due to inconsistent serialization capabilities of _LangChain_ apps. ::: trulens_eval.feedback.provider.langchain.Langchain diff --git a/docs/trulens_eval/contributing/design.md b/docs/trulens_eval/contributing/design.md index 632dbd29a..3d3e9020e 100644 --- a/docs/trulens_eval/contributing/design.md +++ b/docs/trulens_eval/contributing/design.md @@ -36,7 +36,7 @@ function [jsonify][trulens_eval.utils.json.jsonify] is the root of this process. Classes inheriting [BaseModel][pydantic.BaseModel] come with serialization to/from json in the form of [model_dump][pydantic.BaseModel.model_dump] and [model_validate][pydantic.BaseModel.model_validate]. We do not use the -serialization to json part of this capability as a lot of langchain components +serialization to json part of this capability as a lot of _LangChain_ components are tripped to fail it with a "will not serialize" message. However, we use make use of pydantic `fields` to enumerate components of an object ourselves saving us from having to filter out irrelevant internals that are not declared as @@ -77,7 +77,7 @@ various classes. ##### pydantic (langchain) -Most if not all langchain components use pydantic which imposes some +Most if not all _LangChain_ components use pydantic which imposes some restrictions but also provides some utilities. Classes inheriting [BaseModel][pydantic.BaseModel] do not allow defining new attributes but existing attributes including those provided by pydantic itself can be @@ -196,7 +196,7 @@ functions that seem to not involve [Task][asyncio.Task] do use tasks, such as TODO(piotrm): This might have been fixed. Check. - Some apps cannot be serialized/jsonized. Sequential app is an example. This is - a limitation of langchain itself. + a limitation of _LangChain_ itself. - Instrumentation relies on CPython specifics, making heavy use of the [inspect][] module which is not expected to work with other Python @@ -235,7 +235,7 @@ stack for specific frames: #### Alternatives -- [contextvars][] -- langchain uses these to manage contexts such as those used +- [contextvars][] -- _LangChain_ uses these to manage contexts such as those used for instrumenting/tracking LLM usage. These can be used to manage call stack information like we do. The drawback is that these are not threadsafe or at least need instrumenting thread creation. We have to do a similar thing by diff --git a/docs/trulens_eval/contributing/standards.md b/docs/trulens_eval/contributing/standards.md index e010fe5ec..90f1d15d3 100644 --- a/docs/trulens_eval/contributing/standards.md +++ b/docs/trulens_eval/contributing/standards.md @@ -5,15 +5,28 @@ Enumerations of standards for code and its documentation to be maintained in ## Proper Names -Styling/formatting of proper names in italics. +In natural language text, style/format proper names using italics if available. +In Markdown, this can be done with a single underscore character on both sides +of the term. In unstyled text, use the capitalization as below. This does not +apply when referring to things like package names, classes, methods. -- _TruLens_ +- _TruLens_, _TruLens-Eval_, _TruLens-Explain_ - _LangChain_ - _LlamaIndex_ -- _NeMo Guardrails_, _Guardrails_ for short, _rails_ for shorter. +- _NeMo Guardrails_ + +- _OpenAI_ + +- _Bedrock_ + +- _LiteLLM_ + +- _Pinecone_ + +- _HuggingFace_ ## Python diff --git a/docs/trulens_eval/evaluation/feedback_providers/index.md b/docs/trulens_eval/evaluation/feedback_providers/index.md index 3d9425c19..4d446eacb 100644 --- a/docs/trulens_eval/evaluation/feedback_providers/index.md +++ b/docs/trulens_eval/evaluation/feedback_providers/index.md @@ -23,7 +23,7 @@ Providers which use large language models for feedback evaluation: [AzureOpenAI provider][trulens_eval.feedback.provider.openai.AzureOpenAI] - [Bedrock provider][trulens_eval.feedback.provider.bedrock.Bedrock] - [LiteLLM provider][trulens_eval.feedback.provider.litellm.LiteLLM] -- [Langchain provider][trulens_eval.feedback.provider.langchain.Langchain] +- [_LangChain_ provider][trulens_eval.feedback.provider.langchain.Langchain] Feedback functions in common across these providers are in their abstract class [LLMProvider][trulens_eval.feedback.provider.base.LLMProvider]. diff --git a/docs/trulens_eval/evaluation/feedback_selectors/selecting_components.md b/docs/trulens_eval/evaluation/feedback_selectors/selecting_components.md index 94864007b..ded9aabeb 100644 --- a/docs/trulens_eval/evaluation/feedback_selectors/selecting_components.md +++ b/docs/trulens_eval/evaluation/feedback_selectors/selecting_components.md @@ -108,11 +108,11 @@ The top level record also contains these helper accessors - `RecordInput = Record.main_input` -- points to the main input part of a Record. This is the first argument to the root method of an app (for - langchain Chains this is the `__call__` method). + _LangChain_ Chains this is the `__call__` method). - `RecordOutput = Record.main_output` -- points to the main output part of a Record. This is the output of the root method of an app (i.e. `__call__` - for langchain Chains). + for _LangChain_ Chains). - `RecordCalls = Record.app` -- points to the root of the app-structured mirror of calls in a record. See **App-organized Calls** Section above. diff --git a/docs/trulens_eval/evaluation/feedback_selectors/selector_shortcuts.md b/docs/trulens_eval/evaluation/feedback_selectors/selector_shortcuts.md index 3dfc303d0..61261b83d 100644 --- a/docs/trulens_eval/evaluation/feedback_selectors/selector_shortcuts.md +++ b/docs/trulens_eval/evaluation/feedback_selectors/selector_shortcuts.md @@ -33,7 +33,7 @@ Several utility methods starting with `.on` provide shorthands: Some wrappers include additional shorthands: -### Llama-Index specific selectors +### LlamaIndex specific selectors - `TruLlama.select_source_nodes()` -- outputs the selector of the source documents part of the engine output. @@ -55,7 +55,7 @@ Some wrappers include additional shorthands: context = TruLlama.select_context(query_engine) ``` -### LangChain specific selectors +### _LangChain_ specific selectors - `TruChain.select_context()` -- outputs the selector of the context part of the engine output. @@ -67,10 +67,10 @@ Some wrappers include additional shorthands: context = TruChain.select_context(retriever_chain) ``` -### Llama-Index and Langchain specific selectors +### _LlamaIndex_ and _LangChain_ specific selectors - `App.select_context()` -- outputs the selector of the context part of the - engine output. Can be used for both Llama-Index and Langchain apps. + engine output. Can be used for both _LlamaIndex_ and _LangChain_ apps. Usage: diff --git a/docs/trulens_eval/getting_started/core_concepts/index.md b/docs/trulens_eval/getting_started/core_concepts/index.md index c10b4cbb0..ee31cdb90 100644 --- a/docs/trulens_eval/getting_started/core_concepts/index.md +++ b/docs/trulens_eval/getting_started/core_concepts/index.md @@ -10,8 +10,11 @@ General and 🦑_TruLens-Eval_-specific concepts. -- `Agent`. A `Component` of an `Application` that performs some related set of - tasks potentially interfacing with some external services or APIs. +- `Agent`. A `Component` of an `Application` or the entirety of an application + that providers a natural language interface to some set of capabilities + typically incorporating `Tools` to invoke or query local or remote services, + while maintaining its state via `Memory`. The user of an agent may be a human, a + tool, or another agent. See also `Multi Agent System`. - `Application` or `App`. An "application" that is tracked by 🦑_TruLens-Eval_. Abstract definition of this tracking corresponds to @@ -26,10 +29,24 @@ General and 🦑_TruLens-Eval_-specific concepts. - `Chain`. A _LangChain_ `App`. +- `Chain of Thought`. The use of an `Agent` to deconstruct its tasks and to + structure, analyze, and refine its `Completions`. + - `Completion`, `Generation`. The process or result of LLM responding to some `Prompt`. -- `Component`. Part of an `Application`. +- `Component`. Part of an `Application` giving it some capability. Typical + components include: + + - `Retriever` + + - `Memory` + + - `Tool` + + - `Prompt Template` + + - `LLM` - `Embedding`. A real vector representation of some piece of text. Can be used to find related pieces of text in a `Retrieval`. @@ -48,11 +65,35 @@ General and 🦑_TruLens-Eval_-specific concepts. - `Human Feedback`. A feedback that is provided by a human, e.g. a thumbs up/down in response to a `Completion`. +- `Instruction Prompt`, `System Prompt`. A part of a `Prompt` given to an `LLM` + to complete that contains instructions describing the task that the + `Completion` should solve. Sometimes such prompts include examples of correct + or desirable completions (see `Shots`). A prompt that does not include examples + is said to be `Zero Shot`. + - `LLM`, `Large Language Model`. The `Component` of an `Application` that performs `Completion`. +- `Memory`. The state maintained by an `Application` or an `Agent` indicating + anything relevant to continuing, refining, or guiding it towards its + goals. `Memory` is provided as `Context` in `Prompts` and is updated when new + relevant context is processed, be it a user prompt or the results of the + invocation of some `Tool`. As `Memory` is included in `Prompts`, it can be a + natural language description of the state of the app/agent. To limit to size + if memory, `Summarization` is often used. + +- `Multi-Agent System`. The use of multiple `Agents` incentivized to interact + with each other to implement some capability. While the term predates `LLMs`, + the convenience of the common natural language interface makes the approach + much easier to implement. + - `Prompt`. The text that an `LLM` completes during `Completion`. In chat - applications, the user's message. + applications. See also `Instruction Prompt`, `Prompt Template`. + +- `Prompt Template`. A piece of text with placeholders to be filled in in order + to build a `Prompt` for a given task. A `Prompt Template` will typically + include the `Instruction Prompt` with placeholders for things like `Context`, + `Memory`, or `Application` configuration parameters. - `Provider`. A system that _provides_ the ability to execute models, either `LLM`s or classification models. In 🦑_TruLens-Eval_, `Feedback Functions` @@ -73,18 +114,36 @@ General and 🦑_TruLens-Eval_-specific concepts. !!! note This will be renamed to `Trace` in the future. -- `Retrieval`. The process or result of looking up pieces of context relevant to - some query. Typically this is done using an `Embedding` reprqesentations of - queries and contexts. +- `Retrieval`, `Retriever`. The process or result (or the `Component` that + performs this) of looking up pieces of text relevant to a `Prompt` to provide + as `Context` to an `LLM`. Typically this is done using an `Embedding` + representations. - `Selector` (🦑_TruLens-Eval_-specific concept). A specification of the source of data from a `Trace` to use as inputs to a `Feedback Function`. This corresponds to [Lens][trulens_eval.utils.serial.Lens] and utilities [Select][trulens_eval.schema.Select]. +- `Shot`, `Zero Shot`, `Few Shot`, `-Shot`. The use of zero or more + examples in an `Instruction Prompt` to help an `LLM` generate desirable + `Completions`. `Zero Shot` describes prompts that do not have any examples and + only offer a natural language description of the task, while `-Shot` + indicate some `` of examples are provided. + - `Span`. Some unit of work logged as part of a record. Corresponds to current 🦑[RecordAppCallMethod][trulens_eval.schema.RecordAppCall]. -- `Tool`. See `Agent`. +- `Summarization`. The task of condensing some natural language text into a + smaller bit of natural language text that preserves the most important parts + of the text. This can be targetted towards humans or otherwise. It can also be + used to maintain consize `Memory` in an `LLM` `Application` or `Agent`. + Summarization can be performed by an `LLM` using a specific `Instruction Prompt`. + +- `Tool`. A piece of functionality that can be invoked by an `Application` or + `Agent`. This commonly includes interfaces to services such as search (generic + search via google or more specific like IMDB for movies). Tools may also + perform actions such as submitting comments to github issues. A `Tool` may + also encapsulate an interface to an `Agent` for use as a component in a larger + `Application`. - `Trace`. See `Record`. diff --git a/docs/trulens_eval/tracking/instrumentation/index.ipynb b/docs/trulens_eval/tracking/instrumentation/index.ipynb index beb2dec95..594e6d46b 100644 --- a/docs/trulens_eval/tracking/instrumentation/index.ipynb +++ b/docs/trulens_eval/tracking/instrumentation/index.ipynb @@ -18,7 +18,7 @@ " LLM app.\n", "* TruChain instruments LangChain apps. [Read\n", " more](langchain).\n", - "* TruLlama instruments Llama-Index apps. [Read\n", + "* TruLlama instruments LlamaIndex apps. [Read\n", " more](llama_index).\n", "* TruRails instruments NVIDIA Nemo Guardrails apps. [Read more](nemo).\n", "\n", diff --git a/docs/trulens_eval/tracking/instrumentation/langchain.ipynb b/docs/trulens_eval/tracking/instrumentation/langchain.ipynb index acbb3a9b7..2b349bb42 100644 --- a/docs/trulens_eval/tracking/instrumentation/langchain.ipynb +++ b/docs/trulens_eval/tracking/instrumentation/langchain.ipynb @@ -4,12 +4,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 📓 🦜️🔗 LangChain Integration\n", + "# 📓 🦜️🔗 _LangChain_ Integration\n", "\n", - "TruLens provides TruChain, a deep integration with LangChain to allow you to\n", - "inspect and evaluate the internals of your application built using LangChain.\n", - "This is done through the instrumentation of key LangChain classes. To see a list\n", - "of classes instrumented, see *Appendix: Instrumented LangChain Classes and\n", + "TruLens provides TruChain, a deep integration with _LangChain_ to allow you to\n", + "inspect and evaluate the internals of your application built using _LangChain_.\n", + "This is done through the instrumentation of key _LangChain_ classes. To see a list\n", + "of classes instrumented, see *Appendix: Instrumented _LangChain_ Classes and\n", "Methods*.\n", "\n", "In addition to the default instrumentation, TruChain exposes the\n", @@ -41,7 +41,7 @@ "from langchain.prompts.chat import HumanMessagePromptTemplate, ChatPromptTemplate\n", "from trulens_eval import TruChain\n", "\n", - "# typical langchain rag setup\n", + "# typical LangChain rag setup\n", "full_prompt = HumanMessagePromptTemplate(\n", " prompt=PromptTemplate(\n", " template=\n", @@ -157,7 +157,7 @@ "source": [ "## Async Support\n", "\n", - "TruChain also provides async support for Langchain through the `acall` method. This allows you to track and evaluate async and streaming LangChain applications.\n", + "TruChain also provides async support for _LangChain_ through the `acall` method. This allows you to track and evaluate async and streaming _LangChain_ applications.\n", "\n", "As an example, below is an LLM chain set up with an async callback." ] diff --git a/docs/trulens_eval/tracking/instrumentation/llama_index.ipynb b/docs/trulens_eval/tracking/instrumentation/llama_index.ipynb index df2e72892..dd54ac567 100644 --- a/docs/trulens_eval/tracking/instrumentation/llama_index.ipynb +++ b/docs/trulens_eval/tracking/instrumentation/llama_index.ipynb @@ -4,12 +4,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 📓 🦙 Llama-Index Integration\n", + "# 📓 🦙 LlamaIndex Integration\n", "\n", - "TruLens provides TruLlama, a deep integration with Llama-Index to allow you to\n", - "inspect and evaluate the internals of your application built using Llama-Index.\n", - "This is done through the instrumentation of key Llama-Index classes and methods.\n", - "To see all classes and methods instrumented, see *Appendix: Llama-Index\n", + "TruLens provides TruLlama, a deep integration with LlamaIndex to allow you to\n", + "inspect and evaluate the internals of your application built using LlamaIndex.\n", + "This is done through the instrumentation of key LlamaIndex classes and methods.\n", + "To see all classes and methods instrumented, see *Appendix: LlamaIndex\n", "Instrumented Classes and Methods*.\n", "\n", "In addition to the default instrumentation, TruChain exposes the\n", @@ -25,7 +25,7 @@ "source": [ "## Example usage\n", "\n", - "Below is a quick example of usage. First, we'll create a standard Llama-Index query engine from Paul Graham's Essay, *What I Worked On* " + "Below is a quick example of usage. First, we'll create a standard LlamaIndex query engine from Paul Graham's Essay, *What I Worked On* " ] }, { @@ -49,7 +49,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To instrument an Llama-Index query engine, all that's required is to wrap it using TruLlama." + "To instrument an LlamaIndex query engine, all that's required is to wrap it using TruLlama." ] }, { @@ -83,7 +83,7 @@ "internal step of our application, such as the retreived context. Doing so allows\n", "us to evaluate for metrics including context relevance and groundedness.\n", "\n", - "For Llama-Index applications where the source nodes are used, `select_context`\n", + "For LlamaIndex applications where the source nodes are used, `select_context`\n", "can be used to access the retrieved text for evaluation.\n", "\n", "Example usage:\n", @@ -113,7 +113,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You can find the full quickstart available here: [Llama-Index Quickstart](/trulens_eval/llama_index_quickstart)" + "You can find the full quickstart available here: [LlamaIndex Quickstart](/trulens_eval/llama_index_quickstart)" ] }, { @@ -121,11 +121,11 @@ "metadata": {}, "source": [ "## Async Support\n", - "TruLlama also provides async support for Llama-Index through the `aquery`,\n", + "TruLlama also provides async support for LlamaIndex through the `aquery`,\n", "`achat`, and `astream_chat` methods. This allows you to track and evaluate async\n", "applciations.\n", "\n", - "As an example, below is an Llama-Index async chat engine (`achat`)." + "As an example, below is an LlamaIndex async chat engine (`achat`)." ] }, { @@ -153,7 +153,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To instrument an Llama-Index `achat` engine, all that's required is to wrap it using TruLlama - just like with the query engine." + "To instrument an LlamaIndex `achat` engine, all that's required is to wrap it using TruLlama - just like with the query engine." ] }, { @@ -196,9 +196,9 @@ "source": [ "## Streaming Support\n", "\n", - "TruLlama also provides streaming support for Llama-Index. This allows you to track and evaluate streaming applications.\n", + "TruLlama also provides streaming support for LlamaIndex. This allows you to track and evaluate streaming applications.\n", "\n", - "As an example, below is an Llama-Index query engine with streaming." + "As an example, below is an LlamaIndex query engine with streaming." ] }, { @@ -260,14 +260,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For more usage examples, check out the [Llama-Index examples directory](https://github.com/truera/trulens/tree/main/trulens_eval/examples/frameworks/llama_index)." + "For more usage examples, check out the [LlamaIndex examples directory](https://github.com/truera/trulens/tree/main/trulens_eval/examples/frameworks/llama_index)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Appendix: Llama-Index Instrumented Classes and Methods\n", + "## Appendix: LlamaIndex Instrumented Classes and Methods\n", "\n", "The modules, classes, and methods that trulens instruments can be retrieved from\n", "the appropriate Instrument subclass." diff --git a/docs/trulens_eval/tracking/instrumentation/nemo.ipynb b/docs/trulens_eval/tracking/instrumentation/nemo.ipynb index 3717046ea..aac381017 100644 --- a/docs/trulens_eval/tracking/instrumentation/nemo.ipynb +++ b/docs/trulens_eval/tracking/instrumentation/nemo.ipynb @@ -4,11 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 📓 NeMo Guardrails Integration\n", + "# 📓 _NeMo Guardrails_ Integration\n", "\n", - "TruLens provides TruRails, an integration with NeMo Guardrails apps to allow you to\n", - "inspect and evaluate the internals of your application built using nemo.\n", - "This is done through the instrumentation of key nemo classes. To see a list\n", + "TruLens provides TruRails, an integration with _NeMo Guardrails_ apps to allow you to\n", + "inspect and evaluate the internals of your application built using _NeMo Guardrails_.\n", + "This is done through the instrumentation of key _NeMo Guardrails_ classes. To see a list\n", "of classes instrumented, see *Appendix: Instrumented Nemo Classes and\n", "Methods*.\n", "\n", diff --git a/mkdocs.yml b/mkdocs.yml index 786a05054..7802a5584 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -250,7 +250,7 @@ nav: - AzureOpenAI: trulens_eval/api/provider/openai/azureopenai.md - AWS Bedrock: trulens_eval/api/provider/bedrock.md - LiteLLM: trulens_eval/api/provider/litellm.md - - 🦜️🔗 Langchain: trulens_eval/api/provider/langchain.md + - 🦜️🔗 LangChain: trulens_eval/api/provider/langchain.md - 🤗 HuggingFace: trulens_eval/api/provider/huggingface.md - Endpoint: - trulens_eval/api/endpoint/index.md diff --git a/trulens_eval/examples/experimental/db_populate.ipynb b/trulens_eval/examples/experimental/db_populate.ipynb index 6b73c74c2..9dd6b4472 100644 --- a/trulens_eval/examples/experimental/db_populate.ipynb +++ b/trulens_eval/examples/experimental/db_populate.ipynb @@ -183,7 +183,7 @@ "source": [ "# Bedrock endpoint\n", "# Cohere as endpoint\n", - "# Langchain as endpoint\n", + "# LangChain as endpoint\n", "# Litellm as endpoint" ] }, @@ -200,7 +200,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Langchain app" + "# _LangChain_ app" ] }, { diff --git a/trulens_eval/examples/experimental/generate_test_set.ipynb b/trulens_eval/examples/experimental/generate_test_set.ipynb index b562039d8..09ce47dfa 100644 --- a/trulens_eval/examples/experimental/generate_test_set.ipynb +++ b/trulens_eval/examples/experimental/generate_test_set.ipynb @@ -52,7 +52,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Imports from langchain to build app\n", + "# Imports from LangChain to build app\n", "import bs4\n", "from langchain import hub\n", "from langchain.chat_models import ChatOpenAI\n", diff --git a/trulens_eval/examples/experimental/virtual_example.ipynb b/trulens_eval/examples/experimental/virtual_example.ipynb index 6f2826118..7a4ac239b 100644 --- a/trulens_eval/examples/experimental/virtual_example.ipynb +++ b/trulens_eval/examples/experimental/virtual_example.ipynb @@ -161,7 +161,7 @@ "metadata": {}, "outputs": [], "source": [ - "# The same feedback function as the langchain quickstart except the selector for\n", + "# The same feedback function as the LangChain quickstart except the selector for\n", "# context is different.\n", "\n", "from trulens_eval.feedback.provider import OpenAI\n", diff --git a/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example.ipynb b/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example.ipynb index e16408817..3a897e4cd 100644 --- a/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example.ipynb +++ b/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example.ipynb @@ -104,7 +104,7 @@ "import os\n", "from pprint import PrettyPrinter\n", "\n", - "# Imports from langchain to build app:\n", + "# Imports from LangChain to build app:\n", "from langchain.chains import ConversationalRetrievalChain\n", "from langchain.embeddings.openai import OpenAIEmbeddings\n", "from langchain_community.llms import OpenAI\n", @@ -245,7 +245,7 @@ "source": [ "def v1_new_conversation(feedback_mode=FeedbackMode.WITH_APP):\n", " \"\"\"\n", - " Create a langchain app for a new conversation with a question-answering bot.\n", + " Create a _LangChain_ app for a new conversation with a question-answering bot.\n", "\n", " Feedback_mode controls when feedback is evaluated:\n", "\n", @@ -378,7 +378,7 @@ "source": [ "def v2_new_conversation(feedback_mode=FeedbackMode.WITH_APP):\n", " \"\"\"\n", - " Create a langchain app for a new conversation with a question-answering bot.\n", + " Create a _LangChain_ app for a new conversation with a question-answering bot.\n", " \"\"\"\n", "\n", " # Blank conversation memory.\n", @@ -481,7 +481,7 @@ "source": [ "def v3_new_conversation(feedback_mode=FeedbackMode.WITH_APP):\n", " \"\"\"\n", - " Create a langchain app for a new conversation with a question-answering bot.\n", + " Create a _LangChain_ app for a new conversation with a question-answering bot.\n", " \"\"\"\n", "\n", " # Blank conversation memory.\n", @@ -565,7 +565,7 @@ "source": [ "def v4_new_conversation(feedback_mode=FeedbackMode.WITH_APP):\n", " \"\"\"\n", - " Create a langchain app for a new conversation with a question-answering bot.\n", + " Create a _LangChain_ app for a new conversation with a question-answering bot.\n", " \"\"\"\n", "\n", " ### TO FILL IN HERE ###\n", diff --git a/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example_no_hugs.ipynb b/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example_no_hugs.ipynb index 3b46be4e0..2e67204fd 100644 --- a/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example_no_hugs.ipynb +++ b/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example_no_hugs.ipynb @@ -64,7 +64,7 @@ "source": [ "from pprint import PrettyPrinter\n", "\n", - "# Imports from langchain to build app:\n", + "# Imports from LangChain to build app:\n", "from langchain.chains import ConversationalRetrievalChain\n", "from langchain.embeddings.openai import OpenAIEmbeddings\n", "from langchain_community.llms import OpenAI\n", @@ -209,7 +209,7 @@ "source": [ "def v1_new_conversation(feedback_mode=FeedbackMode.WITH_APP):\n", " \"\"\"\n", - " Create a langchain app for a new conversation with a question-answering bot.\n", + " Create a _LangChain_ app for a new conversation with a question-answering bot.\n", "\n", " Feedback_mode controls when feedback is evaluated:\n", "\n", diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_agents.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_agents.ipynb index 0dea95ff0..9cc7f9db6 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_agents.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_agents.ipynb @@ -405,7 +405,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.8.16" }, "vscode": { "interpreter": { diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_async.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_async.ipynb index f930d6a4f..d16b53327 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_async.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_async.ipynb @@ -5,11 +5,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Langchain Async\n", + "# _LangChain_ Async\n", "\n", "One of the biggest pain-points developers discuss when trying to build useful LLM applications is latency; these applications often make multiple calls to LLM APIs, each one taking a few seconds. It can be quite a frustrating user experience to stare at a loading spinner for more than a couple seconds. Streaming helps reduce this perceived latency by returning the output of the LLM token by token, instead of all at once.\n", "\n", - "This notebook demonstrates how to monitor a LangChain streaming app with TruLens.\n", + "This notebook demonstrates how to monitor a _LangChain_ streaming app with TruLens.\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/expositional/frameworks/langchain/langchain_async.ipynb)" ] @@ -241,7 +241,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.11.6" }, "orig_nbformat": 4, "vscode": { diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_ensemble_retriever.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_ensemble_retriever.ipynb index 740069ef4..2d5c17e4d 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_ensemble_retriever.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_ensemble_retriever.ipynb @@ -5,9 +5,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Langchain Ensemble Retriever\n", + "# _LangChain_ Ensemble Retriever\n", "\n", - "The Langchain EnsembleRetriever takes a list of retrievers as input and ensemble the results of their get_relevant_documents() methods and rerank the results based on the Reciprocal Rank Fusion algorithm. With TruLens, we have the ability to evaluate the context of each component retriever along with the ensemble retriever. This example walks through that process.\n", + "The _LangChain_ EnsembleRetriever takes a list of retrievers as input and ensemble the results of their get_relevant_documents() methods and rerank the results based on the Reciprocal Rank Fusion algorithm. With TruLens, we have the ability to evaluate the context of each component retriever along with the ensemble retriever. This example walks through that process.\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/expositional/frameworks/langchain/langchain_ensemble_retriever.ipynb)" ] @@ -51,7 +51,7 @@ "tru = Tru()\n", "tru.reset_database()\n", "\n", - "# Imports from langchain to build app\n", + "# Imports from LangChain to build app\n", "from langchain.retrievers import BM25Retriever, EnsembleRetriever\n", "from langchain_community.vectorstores import FAISS\n", "from langchain_openai import OpenAIEmbeddings" diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_groundtruth.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_groundtruth.ipynb index 54b294f90..88ee0119e 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_groundtruth.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_groundtruth.ipynb @@ -7,7 +7,7 @@ "source": [ "# Ground Truth Evaluations\n", "\n", - "In this quickstart you will create a evaluate a LangChain app using ground truth. Ground truth evaluation can be especially useful during early LLM experiments when you have a small set of example queries that are critical to get right.\n", + "In this quickstart you will create a evaluate a _LangChain_ app using ground truth. Ground truth evaluation can be especially useful during early LLM experiments when you have a small set of example queries that are critical to get right.\n", "\n", "Ground truth evaluation works by comparing the similarity of an LLM response compared to its matching verified response.\n", "\n", diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_retrieval_agent.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_retrieval_agent.ipynb index 713ad7845..ecdf8adef 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_retrieval_agent.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_retrieval_agent.ipynb @@ -4,8 +4,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Langchain retrieval agent \n", - "In this notebook, we are building a Langchain agent to take in user input and figure out the best tool(s) to use via chain of thought (CoT) reasoning. \n", + "# _LangChain_ retrieval agent \n", + "In this notebook, we are building a _LangChain_ agent to take in user input and figure out the best tool(s) to use via chain of thought (CoT) reasoning. \n", "\n", "Given we have more than one distinct tasks defined in the tools for our agent, one being summarization and another one, which generates multiple choice questions and corresponding answers, being more similar to traditional Natural Language Understanding (NLU), we will use to key evaluations for our agent: Tool Input and Tool Selection. Both will be defined with custom functions.\n", "\n", diff --git a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_agents.ipynb b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_agents.ipynb index a1400c2db..20960456d 100644 --- a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_agents.ipynb +++ b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_agents.ipynb @@ -6,7 +6,7 @@ "id": "7PF9uQB4X2gJ" }, "source": [ - "## Llama-Index Agents + Ground Truth & Custom Evaluations\n", + "## LlamaIndex Agents + Ground Truth & Custom Evaluations\n", "\n", "In this example, we build an agent-based app with Llama Index to answer questions with the help of Yelp. We'll evaluate it using a few different feedback functions (some custom, some out-of-the-box)\n", "\n", diff --git a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_async.ipynb b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_async.ipynb index 618d51dba..c8d399c7e 100644 --- a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_async.ipynb +++ b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_async.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Llama-Index Async\n", + "# LlamaIndex Async\n", "\n", "Async is a growing method for LLM applications and can be especially useful for reducing indexing time. This notebook demonstrates how to monitor Llama-index async apps with TruLens." ] @@ -15,7 +15,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Import from Llama-Index and TruLens" + "### Import from LlamaIndex and TruLens" ] }, { diff --git a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_groundtruth.ipynb b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_groundtruth.ipynb index 8ccb66781..938266313 100644 --- a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_groundtruth.ipynb +++ b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_groundtruth.ipynb @@ -5,11 +5,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Groundtruth evaluation for Llama-Index applications\n", + "# Groundtruth evaluation for LlamaIndex applications\n", "\n", "Ground truth evaluation can be especially useful during early LLM experiments when you have a small set of example queries that are critical to get right. Ground truth evaluation works by comparing the similarity of an LLM response compared to its matching verified response.\n", "\n", - "This example walks through how to set up ground truth eval for a Llama-Index app.\n", + "This example walks through how to set up ground truth eval for a LlamaIndex app.\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_groundtruth.ipynb)" ] @@ -19,7 +19,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### import from TruLens and Llama-Index" + "### import from TruLens and LlamaIndex" ] }, { diff --git a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_hybrid_retriever.ipynb b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_hybrid_retriever.ipynb index 85dc44c33..c97bbe460 100644 --- a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_hybrid_retriever.ipynb +++ b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_hybrid_retriever.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Llama-Index Hybrid Retriever + Reranking\n", + "# LlamaIndex Hybrid Retriever + Reranking\n", "\n", "Hybrid Retrievers are a great way to combine the strenghts of different retrievers. Combined with filtering and reranking, this can be especially powerful in retrieving only the most relevant context from multiple methods. TruLens can take us even farther to highlight the strengths of each component retriever along with measuring the success of the hybrid retriever. This example walks through that process.\n", "\n", diff --git a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_multimodal.ipynb b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_multimodal.ipynb index 1734cca52..eeb80bd89 100644 --- a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_multimodal.ipynb +++ b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_multimodal.ipynb @@ -6,7 +6,7 @@ "source": [ "# Evaluating Multi-Modal RAG\n", "\n", - "In this notebook guide, we’ll demonstrate how to evaluate a Llama-Index Multi-Modal RAG system with TruLens.\n", + "In this notebook guide, we’ll demonstrate how to evaluate a LlamaIndex Multi-Modal RAG system with TruLens.\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_multimodal.ipynb)" ] diff --git a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_queryplanning.ipynb b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_queryplanning.ipynb index 7dc6c6548..bf761b570 100644 --- a/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_queryplanning.ipynb +++ b/trulens_eval/examples/expositional/frameworks/llama_index/llama_index_queryplanning.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Query Planning in Llama-Index\n", + "# Query Planning in LlamaIndex\n", "\n", "Query planning is a useful tool to leverage the ability of LLMs to structure the user inputs into multiple different queries, either sequentially or in parallel before answering the questions. This method improvers the response by allowing the question to be decomposed into smaller, more answerable questions.\n", "\n", @@ -21,7 +21,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Import from Llama-Index and TruLens" + "### Import from LlamaIndex and TruLens" ] }, { diff --git a/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_custom_action_with_feedback_example.ipynb b/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_custom_action_with_feedback_example.ipynb index c6d358d91..b78db7de5 100644 --- a/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_custom_action_with_feedback_example.ipynb +++ b/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_custom_action_with_feedback_example.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Feedback functions in NeMo Guardrails apps\n", + "# Feedback functions in _NeMo Guardrails_ apps\n", "\n", "This notebook demonstrates how to use feedback functions from within rails apps.\n", "The integration in the other direction, monitoring rails apps using trulens, is\n", @@ -21,7 +21,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Install nemo guardrails if not already installed.\n", + "# Install NeMo Guardrails if not already installed.\n", "! pip install nemoguardrails" ] }, diff --git a/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_feedback_action_example.ipynb b/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_feedback_action_example.ipynb index 372cbc976..3746f9e17 100644 --- a/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_feedback_action_example.ipynb +++ b/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_feedback_action_example.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Feedback functions in NeMo Guardrails apps\n", + "# Feedback functions in _NeMo Guardrails_ apps\n", "\n", "This notebook demonstrates how to use feedback functions from within rails apps.\n", "The integration in the other direction, monitoring rails apps using trulens, is\n", @@ -21,7 +21,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Install nemo guardrails if not already installed.\n", + "# Install NeMo Guardrails if not already installed.\n", "! pip install nemoguardrails" ] }, diff --git a/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_trurails_example.ipynb b/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_trurails_example.ipynb index 9883f600d..40004b974 100644 --- a/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_trurails_example.ipynb +++ b/trulens_eval/examples/expositional/frameworks/nemoguardrails/nemoguardrails_trurails_example.ipynb @@ -4,9 +4,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Monitoring and Evaluating NeMo Guardrails apps\n", + "# Monitoring and Evaluating _NeMo Guardrails_ apps\n", "\n", - "This notebook demonstrates how to instrument NeMo Guardrails apps to monitor\n", + "This notebook demonstrates how to instrument _NeMo Guardrails_ apps to monitor\n", "their invocations and run feedback functions on their final or intermediate\n", "results. The reverse integration, of using trulens within rails apps, is shown\n", "in the other notebook in this folder." @@ -18,7 +18,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Install nemo guardrails if not already installed.\n", + "# Install NeMo Guardrails if not already installed.\n", "! pip install nemoguardrails" ] }, diff --git a/trulens_eval/examples/expositional/models/Vectara_HHEM_evaluator.ipynb b/trulens_eval/examples/expositional/models/Vectara_HHEM_evaluator.ipynb index 4fcec59e6..bcb7a9387 100644 --- a/trulens_eval/examples/expositional/models/Vectara_HHEM_evaluator.ipynb +++ b/trulens_eval/examples/expositional/models/Vectara_HHEM_evaluator.ipynb @@ -43,7 +43,7 @@ "source": [ "### Import Utilities\n", "\n", - "we're using Langchain utilities to facilitate RAG retrieval and demonstrate Vectara's HHEM.\n", + "we're using LangChain utilities to facilitate RAG retrieval and demonstrate Vectara's HHEM.\n", "- run the cells below to get started. " ] }, diff --git a/trulens_eval/examples/expositional/models/azure_openai_langchain.ipynb b/trulens_eval/examples/expositional/models/azure_openai_langchain.ipynb index b157eb2c4..bf50c38f2 100644 --- a/trulens_eval/examples/expositional/models/azure_openai_langchain.ipynb +++ b/trulens_eval/examples/expositional/models/azure_openai_langchain.ipynb @@ -95,7 +95,7 @@ "import os\n", "import bs4 \n", "\n", - "# Langchain imports\n", + "# LangChain imports\n", "from langchain import hub\n", "from langchain.document_loaders import WebBaseLoader\n", "from langchain.schema import StrOutputParser\n", diff --git a/trulens_eval/examples/expositional/models/bedrock.ipynb b/trulens_eval/examples/expositional/models/bedrock.ipynb index f957264a6..67faf12fa 100644 --- a/trulens_eval/examples/expositional/models/bedrock.ipynb +++ b/trulens_eval/examples/expositional/models/bedrock.ipynb @@ -240,7 +240,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.8.16" }, "vscode": { "interpreter": { diff --git a/trulens_eval/examples/expositional/models/gemini_multi_modal.ipynb b/trulens_eval/examples/expositional/models/gemini_multi_modal.ipynb index 8a5222d2e..9fac0a05a 100644 --- a/trulens_eval/examples/expositional/models/gemini_multi_modal.ipynb +++ b/trulens_eval/examples/expositional/models/gemini_multi_modal.ipynb @@ -12,7 +12,7 @@ "\n", "In the second example, learn how to run semantic evaluations on a multi-modal RAG, including the RAG triad.\n", "\n", - "Note: `google-generativeai` is only available for certain countries and regions. Original example attribution: Llama-Index\n", + "Note: `google-generativeai` is only available for certain countries and regions. Original example attribution: LlamaIndex\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/expositional/models/gemini_multi_modal.ipynb)" ] diff --git a/trulens_eval/examples/expositional/vector-dbs/milvus/milvus_simple.ipynb b/trulens_eval/examples/expositional/vector-dbs/milvus/milvus_simple.ipynb index 54a89c1b9..2e4987769 100644 --- a/trulens_eval/examples/expositional/vector-dbs/milvus/milvus_simple.ipynb +++ b/trulens_eval/examples/expositional/vector-dbs/milvus/milvus_simple.ipynb @@ -112,7 +112,7 @@ "source": [ "### Next we want to create our vector store index\n", "\n", - "By default, Llama-Index will do this in memory as follows:" + "By default, LlamaIndex will do this in memory as follows:" ] }, { diff --git a/trulens_eval/examples/expositional/vector-dbs/pinecone/pinecone_quickstart.ipynb b/trulens_eval/examples/expositional/vector-dbs/pinecone/pinecone_quickstart.ipynb index f980cb18b..457aa4cc6 100644 --- a/trulens_eval/examples/expositional/vector-dbs/pinecone/pinecone_quickstart.ipynb +++ b/trulens_eval/examples/expositional/vector-dbs/pinecone/pinecone_quickstart.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Simple Pinecone setup with Llama-Index + Eval\n", + "# Simple Pinecone setup with LlamaIndex + Eval\n", "\n", "In this example you will create a simple Llama Index RAG application and create the vector store in Pinecone. You'll also set up evaluation and logging with TruLens.\n", "\n", diff --git a/trulens_eval/examples/quickstart/groundtruth_evals.ipynb b/trulens_eval/examples/quickstart/groundtruth_evals.ipynb index 3f7c141f5..605361dc1 100644 --- a/trulens_eval/examples/quickstart/groundtruth_evals.ipynb +++ b/trulens_eval/examples/quickstart/groundtruth_evals.ipynb @@ -7,7 +7,7 @@ "source": [ "# 📓 Ground Truth Evaluations\n", "\n", - "In this quickstart you will create a evaluate a LangChain app using ground truth. Ground truth evaluation can be especially useful during early LLM experiments when you have a small set of example queries that are critical to get right.\n", + "In this quickstart you will create a evaluate a _LangChain_ app using ground truth. Ground truth evaluation can be especially useful during early LLM experiments when you have a small set of example queries that are critical to get right.\n", "\n", "Ground truth evaluation works by comparing the similarity of an LLM response compared to its matching verified response.\n", "\n", diff --git a/trulens_eval/examples/quickstart/langchain_quickstart.ipynb b/trulens_eval/examples/quickstart/langchain_quickstart.ipynb index 5b99175a2..e601a0745 100644 --- a/trulens_eval/examples/quickstart/langchain_quickstart.ipynb +++ b/trulens_eval/examples/quickstart/langchain_quickstart.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 📓 Langchain Quickstart\n", + "# 📓 _LangChain_ Quickstart\n", "\n", "In this quickstart you will create a simple LLM Chain and learn how to log it and get feedback on an LLM response.\n", "\n", @@ -60,7 +60,7 @@ "tru = Tru()\n", "tru.reset_database()\n", "\n", - "# Imports from langchain to build app\n", + "# Imports from LangChain to build app\n", "import bs4\n", "from langchain import hub\n", "from langchain.chat_models import ChatOpenAI\n", diff --git a/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb b/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb index 6ee54ace0..d25ed4961 100644 --- a/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb +++ b/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 📓 Llama-Index Quickstart\n", + "# 📓 LlamaIndex Quickstart\n", "\n", "In this quickstart you will create a simple Llama Index app and learn how to log it and get feedback on an LLM response.\n", "\n", diff --git a/trulens_eval/generated_files/all_tools.py b/trulens_eval/generated_files/all_tools.py index d0cf41614..c4b4fe73c 100644 --- a/trulens_eval/generated_files/all_tools.py +++ b/trulens_eval/generated_files/all_tools.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding: utf-8 -# # 📓 Langchain Quickstart +# # 📓 LangChain Quickstart # # In this quickstart you will create a simple LLM Chain and learn how to log it and get feedback on an LLM response. # @@ -261,7 +261,7 @@ def display_call_stack(data): # Note: Feedback functions evaluated in the deferred manner can be seen in the "Progress" page of the TruLens dashboard. -# # 📓 Llama-Index Quickstart +# # 📓 LlamaIndex Quickstart # # In this quickstart you will create a simple Llama Index app and learn how to log it and get feedback on an LLM response. # diff --git a/trulens_eval/trulens_eval/feedback/provider/base.py b/trulens_eval/trulens_eval/feedback/provider/base.py index b8a5da0e4..89a2b756a 100644 --- a/trulens_eval/trulens_eval/feedback/provider/base.py +++ b/trulens_eval/trulens_eval/feedback/provider/base.py @@ -628,7 +628,7 @@ def _langchain_evaluate_with_cot_reasons( def conciseness(self, text: str) -> float: """ Uses chat completion model. A function that completes a template to - check the conciseness of some text. Prompt credit to Langchain Eval. + check the conciseness of some text. Prompt credit to LangChain Eval. !!! example @@ -653,7 +653,7 @@ def conciseness(self, text: str) -> float: def conciseness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: """ Uses chat completion model. A function that completes a template to - check the conciseness of some text. Prompt credit to Langchain Eval. + check the conciseness of some text. Prompt credit to LangChain Eval. !!! example @@ -679,7 +679,7 @@ def conciseness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: def correctness(self, text: str) -> float: """ Uses chat completion model. A function that completes a template to - check the correctness of some text. Prompt credit to Langchain Eval. + check the correctness of some text. Prompt credit to LangChain Eval. !!! example @@ -703,7 +703,7 @@ def correctness(self, text: str) -> float: def correctness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: """ Uses chat completion model. A function that completes a template to - check the correctness of some text. Prompt credit to Langchain Eval. + check the correctness of some text. Prompt credit to LangChain Eval. Also uses chain of thought methodology and emits the reasons. !!! example @@ -728,7 +728,7 @@ def correctness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: def coherence(self, text: str) -> float: """ Uses chat completion model. A function that completes a - template to check the coherence of some text. Prompt credit to Langchain Eval. + template to check the coherence of some text. Prompt credit to LangChain Eval. !!! example @@ -752,7 +752,7 @@ def coherence(self, text: str) -> float: def coherence_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: """ Uses chat completion model. A function that completes a template to - check the coherence of some text. Prompt credit to Langchain Eval. Also + check the coherence of some text. Prompt credit to LangChain Eval. Also uses chain of thought methodology and emits the reasons. !!! example @@ -777,7 +777,7 @@ def coherence_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: def harmfulness(self, text: str) -> float: """ Uses chat completion model. A function that completes a template to - check the harmfulness of some text. Prompt credit to Langchain Eval. + check the harmfulness of some text. Prompt credit to LangChain Eval. !!! example @@ -801,7 +801,7 @@ def harmfulness(self, text: str) -> float: def harmfulness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: """ Uses chat completion model. A function that completes a template to - check the harmfulness of some text. Prompt credit to Langchain Eval. + check the harmfulness of some text. Prompt credit to LangChain Eval. Also uses chain of thought methodology and emits the reasons. !!! example @@ -824,7 +824,7 @@ def harmfulness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: def maliciousness(self, text: str) -> float: """ Uses chat completion model. A function that completes a template to - check the maliciousness of some text. Prompt credit to Langchain Eval. + check the maliciousness of some text. Prompt credit to LangChain Eval. !!! example @@ -849,7 +849,7 @@ def maliciousness(self, text: str) -> float: def maliciousness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: """ Uses chat compoletion model. A function that completes a - template to check the maliciousness of some text. Prompt credit to Langchain Eval. + template to check the maliciousness of some text. Prompt credit to LangChain Eval. Also uses chain of thought methodology and emits the reasons. !!! example @@ -874,7 +874,7 @@ def maliciousness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: def helpfulness(self, text: str) -> float: """ Uses chat completion model. A function that completes a template to - check the helpfulness of some text. Prompt credit to Langchain Eval. + check the helpfulness of some text. Prompt credit to LangChain Eval. !!! example @@ -898,7 +898,7 @@ def helpfulness(self, text: str) -> float: def helpfulness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: """ Uses chat completion model. A function that completes a template to - check the helpfulness of some text. Prompt credit to Langchain Eval. + check the helpfulness of some text. Prompt credit to LangChain Eval. Also uses chain of thought methodology and emits the reasons. !!! example @@ -977,7 +977,7 @@ def controversiality_with_cot_reasons(self, def misogyny(self, text: str) -> float: """ Uses chat completion model. A function that completes a template to - check the misogyny of some text. Prompt credit to Langchain Eval. + check the misogyny of some text. Prompt credit to LangChain Eval. !!! example @@ -1001,7 +1001,7 @@ def misogyny(self, text: str) -> float: def misogyny_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: """ Uses chat completion model. A function that completes a template to - check the misogyny of some text. Prompt credit to Langchain Eval. Also + check the misogyny of some text. Prompt credit to LangChain Eval. Also uses chain of thought methodology and emits the reasons. !!! example @@ -1026,7 +1026,7 @@ def misogyny_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: def criminality(self, text: str) -> float: """ Uses chat completion model. A function that completes a template to - check the criminality of some text. Prompt credit to Langchain Eval. + check the criminality of some text. Prompt credit to LangChain Eval. !!! example @@ -1051,7 +1051,7 @@ def criminality(self, text: str) -> float: def criminality_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: """ Uses chat completion model. A function that completes a template to - check the criminality of some text. Prompt credit to Langchain Eval. + check the criminality of some text. Prompt credit to LangChain Eval. Also uses chain of thought methodology and emits the reasons. !!! example @@ -1076,7 +1076,7 @@ def criminality_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: def insensitivity(self, text: str) -> float: """ Uses chat completion model. A function that completes a template to - check the insensitivity of some text. Prompt credit to Langchain Eval. + check the insensitivity of some text. Prompt credit to LangChain Eval. !!! example @@ -1100,7 +1100,7 @@ def insensitivity(self, text: str) -> float: def insensitivity_with_cot_reasons(self, text: str) -> Tuple[float, Dict]: """ Uses chat completion model. A function that completes a template to - check the insensitivity of some text. Prompt credit to Langchain Eval. + check the insensitivity of some text. Prompt credit to LangChain Eval. Also uses chain of thought methodology and emits the reasons. !!! example diff --git a/trulens_eval/trulens_eval/feedback/provider/endpoint/bedrock.py b/trulens_eval/trulens_eval/feedback/provider/endpoint/bedrock.py index ddaaefba1..969f269e3 100644 --- a/trulens_eval/trulens_eval/feedback/provider/endpoint/bedrock.py +++ b/trulens_eval/trulens_eval/feedback/provider/endpoint/bedrock.py @@ -92,7 +92,7 @@ def handle_generation(self, response: Any) -> None: 'contentType': 'application/json'} """ - # NOTE(piotrm) langchain does not currently support cost tracking for + # NOTE(piotrm) LangChain does not currently support cost tracking for # Bedrock. We can at least count successes and tokens visible in the # example output above. diff --git a/trulens_eval/trulens_eval/feedback/provider/endpoint/langchain.py b/trulens_eval/trulens_eval/feedback/provider/endpoint/langchain.py index cf515eaa3..165818073 100644 --- a/trulens_eval/trulens_eval/feedback/provider/endpoint/langchain.py +++ b/trulens_eval/trulens_eval/feedback/provider/endpoint/langchain.py @@ -24,7 +24,7 @@ def handle_generation(self, response: Any) -> None: class LangchainEndpoint(Endpoint): """ - Langchain endpoint. + LangChain endpoint. """ # Cannot validate BaseLLM / BaseChatModel as they are pydantic v1 and there diff --git a/trulens_eval/trulens_eval/feedback/provider/langchain.py b/trulens_eval/trulens_eval/feedback/provider/langchain.py index e52614ece..29c06e24b 100644 --- a/trulens_eval/trulens_eval/feedback/provider/langchain.py +++ b/trulens_eval/trulens_eval/feedback/provider/langchain.py @@ -12,9 +12,9 @@ class Langchain(LLMProvider): - """Out of the box feedback functions using Langchain LLMs and ChatModels + """Out of the box feedback functions using LangChain LLMs and ChatModels - Create a Langchain Provider with out of the box feedback functions. + Create a LangChain Provider with out of the box feedback functions. !!! example @@ -27,7 +27,7 @@ class Langchain(LLMProvider): ``` Args: - chain: Langchain LLM. + chain: LangChain LLM. """ endpoint: LangchainEndpoint diff --git a/trulens_eval/trulens_eval/requirements.optional.txt b/trulens_eval/trulens_eval/requirements.optional.txt index 75d0a16e3..b5fce0d23 100644 --- a/trulens_eval/trulens_eval/requirements.optional.txt +++ b/trulens_eval/trulens_eval/requirements.optional.txt @@ -8,9 +8,9 @@ bert-score >= 0.3.13 # groundtruth.py evaluate >= 0.4.0 # same nltk >= 3.8.1 # groundedness.py -# Apps: Langchain (optional parts) +# Apps: LangChain (optional parts) langchainhub >= 0.1.14 # example notebooks -langchain_community >= 0.0.17 # example notebooks, now required for pinecone, others +langchain_community >= 0.0.17 # example notebooks, now required for Pinecone, others langchain_openai >= 0.0.5 # unit tests, sample apps # Apps: LlamaIndex @@ -19,14 +19,14 @@ nbconvert >= 7.14.2 # for llama_index example using SimlpleDirectoryReader llama-index-readers-file >= 0.1.5 # tests and notebooks llama-index-readers-web >= 0.1.6 # tests and notebooks -# Apps: NEMO Guardrails +# Apps: NeMo Guardrails nemoguardrails >= 0.7.1 -# Models: openai +# Models: OpenAI openai >= 1.1.1, <2 httpx >= 0.23.3 # for openai client serialization -# Models: bedrock +# Models: Bedrock boto3 >= 1.33.6 botocore >= 1.33.6 diff --git a/trulens_eval/trulens_eval/tru_chain.py b/trulens_eval/trulens_eval/tru_chain.py index 300e7aee0..3a098ff90 100644 --- a/trulens_eval/trulens_eval/tru_chain.py +++ b/trulens_eval/trulens_eval/tru_chain.py @@ -130,9 +130,9 @@ class TruChain(App): This recorder is designed for LangChain apps, providing a way to instrument, log, and evaluate their behavior. - !!! example "Creating a Langchain RAG application" + !!! example "Creating a LangChain RAG application" - Consider an example Langchain RAG application. For the complete code example, see [LangChain Quickstart](https://www.trulens.org/trulens_eval/getting_started/quickstarts/langchain_quickstart/). + Consider an example LangChain RAG application. For the complete code example, see [LangChain Quickstart](https://www.trulens.org/trulens_eval/getting_started/quickstarts/langchain_quickstart/). ```python from langchain import hub diff --git a/trulens_eval/trulens_eval/tru_llama.py b/trulens_eval/trulens_eval/tru_llama.py index 7f3641b55..2924f563f 100644 --- a/trulens_eval/trulens_eval/tru_llama.py +++ b/trulens_eval/trulens_eval/tru_llama.py @@ -222,13 +222,13 @@ def __init__(self, *args, **kwargs): class TruLlama(App): """ - Recorder for Llama-Index applications. + Recorder for LlamaIndex applications. - This recorder is designed for Llama-Index apps, providing a way to instrument, log, and evaluate their behavior. + This recorder is designed for LlamaIndex apps, providing a way to instrument, log, and evaluate their behavior. - !!! example "Creating a Llama-Index application" + !!! example "Creating a LlamaIndex application" - Consider an example Llama-Index application. For the complete code example, see [Llama-Index Quickstart](https://docs.llamaindex.ai/en/stable/getting_started/starter_example.html). + Consider an example LlamaIndex application. For the complete code example, see [LlamaIndex Quickstart](https://docs.llamaindex.ai/en/stable/getting_started/starter_example.html). ```python from llama_index.core import VectorStoreIndex, SimpleDirectoryReader @@ -278,7 +278,7 @@ class TruLlama(App): Feedback functions can utilize the specific context produced by the application's query engine. This is achieved using the `select_context` method, which then can be used by a feedback selector, such as `on(context)`. - Further information about Llama-Index apps can be found on the [🦙 LlamaIndex Documentation](https://docs.llamaindex.ai/en/stable/) page. + Further information about LlamaIndex apps can be found on the [🦙 LlamaIndex Documentation](https://docs.llamaindex.ai/en/stable/) page. Args: app: A LlamaIndex application. diff --git a/trulens_eval/trulens_eval/tru_rails.py b/trulens_eval/trulens_eval/tru_rails.py index 9635fe996..0f54518a6 100644 --- a/trulens_eval/trulens_eval/tru_rails.py +++ b/trulens_eval/trulens_eval/tru_rails.py @@ -46,7 +46,7 @@ class RailsActionSelect(Select): - """Selector shorthands for NEMO guardrails apps when used for evaluating + """Selector shorthands for _NeMo Guardrails_ apps when used for evaluating feedback in actions. These should not be used for feedback functions given to `TruRails` but @@ -106,9 +106,9 @@ class RailsActionSelect(Select): class FeedbackActions(): - """ - Feedback action action for NEMO guardrails apps. See docstring of method - `feedback`. + """Feedback action action for _NeMo Guardrails_ apps. + + See docstring of method `feedback`. """ @staticmethod @@ -324,7 +324,7 @@ async def feedback_action( class RailsInstrument(Instrument): - """Instrumentation specification for NEMO guardrails apps.""" + """Instrumentation specification for _NeMo Guardrails_ apps.""" class Default: """Default instrumentation specification.""" @@ -332,7 +332,7 @@ class Default: MODULES = {"nemoguardrails"}.union(LangChainInstrument.Default.MODULES) """Modules to instrument by name prefix. - Note that nemo uses langchain internally for some things. + Note that _NeMo Guardrails_ uses _LangChain_ internally for some things. """ CLASSES = lambda: { @@ -375,11 +375,10 @@ def __init__(self, *args, **kwargs): class TruRails(App): - """ - Recorder for apps defined using NEMO guardrails. + """Recorder for apps defined using _NeMo Guardrails_. Args: - app: A nemo guardrails application. + app: A _NeMo Guardrails_ application. """ model_config: ClassVar[dict] = {'arbitrary_types_allowed': True} diff --git a/trulens_eval/trulens_eval/utils/imports.py b/trulens_eval/trulens_eval/utils/imports.py index d7fb018ad..e0aa48ec6 100644 --- a/trulens_eval/trulens_eval/utils/imports.py +++ b/trulens_eval/trulens_eval/utils/imports.py @@ -240,15 +240,15 @@ def format_import_errors( REQUIREMENT_LLAMA = format_import_errors( - 'llama-index', purpose="instrumenting llama_index apps" + 'llama-index', purpose="instrumenting LlamaIndex apps" ) REQUIREMENT_LANGCHAIN = format_import_errors( - 'langchain', purpose="instrumenting langchain apps" + 'langchain', purpose="instrumenting LangChain apps" ) REQUIREMENT_RAILS = format_import_errors( - "nemoguardrails", purpose="instrumenting nemo guardrails apps" + "nemoguardrails", purpose="instrumenting NeMo Guardrails apps" ) REQUIREMENT_PINECONE = format_import_errors( @@ -286,7 +286,7 @@ def format_import_errors( ) REQUIREMENT_NOTEBOOK = format_import_errors( - ["ipython", "ipywidgets"], purpose="using trulens_eval in a notebook" + ["ipython", "ipywidgets"], purpose="using TruLens-Eval in a notebook" )