diff --git a/docs/content/docs/agent-sdk/integrations/meta.json b/docs/content/docs/agent-sdk/integrations/meta.json index 7b7ebb812..f8ad4991f 100644 --- a/docs/content/docs/agent-sdk/integrations/meta.json +++ b/docs/content/docs/agent-sdk/integrations/meta.json @@ -1,4 +1,7 @@ { "title": "Integrations", - "pages": ["hud"] -} + "pages": [ + "hud", + "observability" + ] +} \ No newline at end of file diff --git a/docs/content/docs/agent-sdk/integrations/observability.mdx b/docs/content/docs/agent-sdk/integrations/observability.mdx new file mode 100644 index 000000000..3b1a316ef --- /dev/null +++ b/docs/content/docs/agent-sdk/integrations/observability.mdx @@ -0,0 +1,62 @@ +--- +title: Observability +description: Trace CUA execution steps and sessions +--- + +## Observability + +CUA has a native integration with [Laminar](https://laminar.sh/) – open-source platform for tracing, evals, and labeling of autonomous AI agents. Read more about Laminar in the [Laminar docs](https://docs.lmnr.ai/). + +## Setup + +Register on [Laminar Cloud](https://laminar.sh/) or spin up a [local instance](https://github.com/lmnr-ai/lmnr) and get the key from your project settings. Set the `LMNR_PROJECT_API_KEY` environment variable to your key. + +```bash +pip install lmnr[all] +export LMNR_PROJECT_API_KEY=your-key +``` + +## Usage + +Then, initialize Laminar at the entry point of your application, register Laminar LiteLLM callback, and all steps of CUA will be automatically traced. + +```python +import os + +import litellm + +from agent import ComputerAgent +from computer import Computer +from lmnr import Laminar, LaminarLiteLLMCallback # [!code highlight] + +Laminar.initialize() # [!code highlight] +litellm.callbacks.append(LaminarLiteLLMCallback()) # [!code highlight] + +computer = Computer( + os_type="linux", + provider_type="cloud", + name=os.getenv("CUA_CONTAINER_NAME"), + api_key=os.getenv("CUA_API_KEY"), +) + +agent = ComputerAgent( + model="openai/computer-use-preview", + tools=[computer], +) + +async def main(): + async for step in agent.run("Create a new file called 'test.txt' in the current directory"): + print(step["output"]) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Viewing traces + +You can view traces in the Laminar UI by going to the traces tab in your project. When you select a trace, +you will see all the agent execution steps, including computer actions, LLM calls, and screenshots. + +For each step, you will see the LLM call, the computer action. The computer actions are highlighted in the timeline in yellow. + +Example trace in Laminar showing the litellm.response span and its output. \ No newline at end of file diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index c3517f0aa..af9123e73 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -16,4 +16,4 @@ "---[CodeXml]API Reference---", "...libraries" ] -} +} \ No newline at end of file diff --git a/docs/public/img/laminar_trace_example.png b/docs/public/img/laminar_trace_example.png new file mode 100644 index 000000000..75c3139d7 Binary files /dev/null and b/docs/public/img/laminar_trace_example.png differ