diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 6761d0d..190ae60 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -1,5 +1,10 @@ name: Build distribution files description: 'Build distribution files' +inputs: + package-path: + description: 'Path to the package to build' + required: false + default: '.' outputs: package-hashes: description: "base64-encoded sha256 hashes of distribution files" @@ -10,10 +15,11 @@ runs: steps: - name: Build distribution files shell: bash + working-directory: ${{ inputs.package-path }} run: poetry build - name: Hash build files for provenance id: package-hashes shell: bash - working-directory: ./dist + working-directory: ${{ inputs.package-path }}/dist run: | echo "package-hashes=$(sha256sum * | base64 -w0)" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd8c2dc..a12d321 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,9 +28,24 @@ jobs: - name: Install poetry uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439 - - uses: ./.github/actions/build + - name: Install packages + run: make install + + - name: Build core package + uses: ./.github/actions/build + with: + package-path: packages/core + + - name: Build langchain package + uses: ./.github/actions/build + with: + package-path: packages/langchain + - uses: ./.github/actions/build-docs + - name: Reinstall packages after build + run: make install + - name: Run tests run: make test @@ -58,8 +73,8 @@ jobs: - name: Install poetry uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439 - - name: Install requirements - run: poetry install + - name: Install packages + run: make install - name: Run tests run: make test diff --git a/.github/workflows/manual-publish.yml b/.github/workflows/manual-publish.yml index 9b35bb2..e59739e 100644 --- a/.github/workflows/manual-publish.yml +++ b/.github/workflows/manual-publish.yml @@ -2,20 +2,26 @@ name: Publish Package on: workflow_dispatch: inputs: + package: + description: 'Which package to publish' + required: true + type: choice + options: + - core + - langchain + - both dry_run: description: 'Is this a dry run? If so no package will be published.' type: boolean required: true jobs: - build-publish: + publish-core: + if: ${{ inputs.package == 'core' || inputs.package == 'both' }} runs-on: ubuntu-latest - # Needed to get tokens during publishing. permissions: id-token: write contents: read - outputs: - package-hashes: ${{ steps.build.outputs.package-hashes}} steps: - uses: actions/checkout@v4 @@ -34,20 +40,49 @@ jobs: - uses: ./.github/actions/build id: build + with: + package-path: packages/core - - name: Publish package distributions to PyPI + - name: Publish core package to PyPI if: ${{ inputs.dry_run == false }} - uses: pypa/gh-action-pypi-publish@release/v1 + # https://github.com/pypa/gh-action-pypi-publish/releases/tag/v1.8.13 + uses: pypa/gh-action-pypi-publish@3cc2c35166dfc1e5ea3bb0491ffdeedcaa50d7c with: - password: ${{env.PYPI_AUTH_TOKEN}} + password: ${{ env.PYPI_AUTH_TOKEN }} + packages-dir: packages/core/dist/ - release-provenance: - needs: [ 'build-publish' ] + publish-langchain: + if: ${{ inputs.package == 'langchain' || inputs.package == 'both' }} + runs-on: ubuntu-latest permissions: - actions: read id-token: write - contents: write - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 - with: - base64-subjects: "${{ needs.build-publish.outputs.package-hashes }}" - upload-assets: ${{ !inputs.dry_run }} + contents: read + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Install poetry + uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439 + + - uses: launchdarkly/gh-actions/actions/release-secrets@release-secrets-v1.2.0 + name: 'Get PyPI token' + with: + aws_assume_role: ${{ vars.AWS_ROLE_ARN }} + ssm_parameter_pairs: '/production/common/releasing/pypi/token = PYPI_AUTH_TOKEN' + + - uses: ./.github/actions/build + id: build + with: + package-path: packages/langchain + + - name: Publish langchain package to PyPI + if: ${{ inputs.dry_run == false }} + # Pinned to v1.8.13 (2024-06-14) for security + # https://github.com/pypa/gh-action-pypi-publish/releases/tag/v1.8.13 + uses: pypa/gh-action-pypi-publish@3cc2c35166dfc1e5ea3bb0491ffdeedcaa50d7c + with: + password: ${{ env.PYPI_AUTH_TOKEN }} + packages-dir: packages/langchain/dist/ diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 7176356..8b2e035 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -5,36 +5,40 @@ on: branches: [ main ] jobs: - release-package: + release-please: runs-on: ubuntu-latest permissions: - id-token: write # Needed if using OIDC to get release secrets. - contents: write # Contents and pull-requests are for release-please to make releases. + contents: write pull-requests: write outputs: - release-created: ${{ steps.release.outputs.release_created }} - upload-tag-name: ${{ steps.release.outputs.tag_name }} - package-hashes: ${{ steps.build.outputs.package-hashes}} + releases_created: ${{ steps.release.outputs.releases_created }} + core_release_created: ${{ steps.release.outputs['packages/core--release_created'] }} + langchain_release_created: ${{ steps.release.outputs['packages/langchain--release_created'] }} steps: - uses: googleapis/release-please-action@v4 id: release + with: + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + release-core: + needs: release-please + if: ${{ needs.release-please.outputs.core_release_created == 'true' }} + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + steps: - uses: actions/checkout@v4 - if: ${{ steps.release.outputs.releases_created == 'true' }} - with: - fetch-depth: 0 # If you only need the current version keep this. - uses: actions/setup-python@v5 - if: ${{ steps.release.outputs.releases_created == 'true' }} with: python-version: 3.9 - name: Install poetry - if: ${{ steps.release.outputs.releases_created == 'true' }} uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439 - uses: launchdarkly/gh-actions/actions/release-secrets@release-secrets-v1.2.0 - if: ${{ steps.release.outputs.releases_created == 'true' }} name: 'Get PyPI token' with: aws_assume_role: ${{ vars.AWS_ROLE_ARN }} @@ -42,26 +46,50 @@ jobs: - uses: ./.github/actions/build id: build - if: ${{ steps.release.outputs.releases_created == 'true' }} + with: + package-path: packages/core - uses: ./.github/actions/build-docs - if: ${{ steps.release.outputs.releases_created == 'true' }} - - name: Publish package distributions to PyPI - if: ${{ steps.release.outputs.releases_created == 'true' }} - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Publish core package to PyPI + # https://github.com/pypa/gh-action-pypi-publish/releases/tag/v1.8.13 + uses: pypa/gh-action-pypi-publish@3cc2c35166dfc1e5ea3bb0491ffdeedcaa50d7c with: - password: ${{env.PYPI_AUTH_TOKEN}} + password: ${{ env.PYPI_AUTH_TOKEN }} + packages-dir: packages/core/dist/ - release-provenance: - needs: [ 'release-package' ] - if: ${{ needs.release-package.outputs.release-created == 'true' }} + release-langchain: + needs: release-please + if: ${{ needs.release-please.outputs.langchain_release_created == 'true' }} + runs-on: ubuntu-latest permissions: - actions: read id-token: write contents: write - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 - with: - base64-subjects: "${{ needs.release-package.outputs.package-hashes }}" - upload-assets: true - upload-tag-name: ${{ needs.release-package.outputs.upload-tag-name }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Install poetry + uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439 + + - uses: launchdarkly/gh-actions/actions/release-secrets@release-secrets-v1.2.0 + name: 'Get PyPI token' + with: + aws_assume_role: ${{ vars.AWS_ROLE_ARN }} + ssm_parameter_pairs: '/production/common/releasing/pypi/token = PYPI_AUTH_TOKEN' + + - uses: ./.github/actions/build + id: build + with: + package-path: packages/langchain + + - name: Publish langchain package to PyPI + # Pinned to v1.8.13 (2024-06-14) for security + # https://github.com/pypa/gh-action-pypi-publish/releases/tag/v1.8.13 + uses: pypa/gh-action-pypi-publish@3cc2c35166dfc1e5ea3bb0491ffdeedcaa50d7c + with: + password: ${{ env.PYPI_AUTH_TOKEN }} + packages-dir: packages/langchain/dist/ diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 30b6d45..d41663b 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,4 @@ { - ".": "0.10.1" + "packages/core": "0.10.1", + "packages/langchain": "0.1.0" } diff --git a/Makefile b/Makefile index 791925c..2c5a82c 100644 --- a/Makefile +++ b/Makefile @@ -14,24 +14,60 @@ help: #! Show this help message @grep -h -F '#!' $(MAKEFILE_LIST) | grep -v grep | sed 's/:.*#!/:/' | column -t -s":" .PHONY: install -install: - @poetry install +install: #! Install all packages + @echo "Installing core package..." + @cd packages/core && poetry install + @echo "Installing langchain package..." + @cd packages/langchain && poetry install + +.PHONY: install-core +install-core: #! Install core package only + @cd packages/core && poetry install + +.PHONY: install-langchain +install-langchain: #! Install langchain package only + @cd packages/langchain && poetry install # # Quality control checks # .PHONY: test -test: #! Run unit tests -test: install - @poetry run pytest $(PYTEST_FLAGS) +test: #! Run unit tests for all packages + @echo "Testing core package..." + @cd packages/core && poetry run pytest $(PYTEST_FLAGS) + @echo "Testing langchain package..." + @cd packages/langchain && poetry run pytest $(PYTEST_FLAGS) + +.PHONY: test-core +test-core: #! Run unit tests for core package + @cd packages/core && poetry run pytest $(PYTEST_FLAGS) + +.PHONY: test-langchain +test-langchain: #! Run unit tests for langchain package + @cd packages/langchain && poetry run pytest $(PYTEST_FLAGS) .PHONY: lint lint: #! Run type analysis and linting checks -lint: install - @poetry run mypy ldai - @poetry run isort --check --atomic ldai - @poetry run pycodestyle ldai + @echo "Linting core package..." + @cd packages/core && poetry run mypy ldai + @cd packages/core && poetry run isort --check --atomic ldai + @cd packages/core && poetry run pycodestyle ldai + +.PHONY: build +build: #! Build all packages + @echo "Building core package..." + @cd packages/core && poetry build + @echo "Building langchain package..." + @cd packages/langchain && poetry build + +.PHONY: build-core +build-core: #! Build core package + @cd packages/core && poetry build + +.PHONY: build-langchain +build-langchain: #! Build langchain package + @cd packages/langchain && poetry build # # Documentation generation @@ -39,6 +75,5 @@ lint: install .PHONY: docs docs: #! Generate sphinx-based documentation - @poetry install --with docs - @cd docs - @poetry run $(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @cd packages/core && poetry install --with docs + @cd packages/core && poetry run $(SPHINXBUILD) -M html "../../$(SOURCEDIR)" "../../$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/PROVENANCE.md b/PROVENANCE.md index 73d317c..4a20571 100644 --- a/PROVENANCE.md +++ b/PROVENANCE.md @@ -4,38 +4,62 @@ LaunchDarkly uses the [SLSA framework](https://slsa.dev/spec/v1.0/about) (Supply As part of [SLSA requirements for level 3 compliance](https://slsa.dev/spec/v1.0/requirements), LaunchDarkly publishes provenance about our SDK package builds using [GitHub's generic SLSA3 provenance generator](https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md#generation-of-slsa3-provenance-for-arbitrary-projects) for distribution alongside our packages. These attestations are available for download from the GitHub release page for the release version under Assets > `multiple.intoto.jsonl`. -To verify SLSA provenance attestations, we recommend using [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Example usage for verifying a package is included below: +To verify SLSA provenance attestations, we recommend using [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Example usage for verifying packages is included below. + +### Verifying the Core Package -``` -# Set the version of the library to verify -VERSION=0.10.1 +```bash +# Set the version of the core package to verify +CORE_VERSION=0.10.1 ``` +```bash +# Download package from PyPI +$ pip download --only-binary=:all: launchdarkly-server-sdk-ai==${CORE_VERSION} + +# Download provenance from GitHub release into same directory +$ curl --location -O \ + https://github.com/launchdarkly/python-server-sdk-ai/releases/download/core-${CORE_VERSION}/multiple.intoto.jsonl + +# Run slsa-verifier to verify provenance against package artifacts +$ slsa-verifier verify-artifact \ +--provenance-path multiple.intoto.jsonl \ +--source-uri github.com/launchdarkly/python-server-sdk-ai \ +launchdarkly_server_sdk_ai-${CORE_VERSION}-py3-none-any.whl ``` -# Download package from PyPi -$ pip download --only-binary=:all: launchdarkly-server-sdk-ai==${VERSION} -# Download provenance from Github release into same directory +### Verifying the LangChain Package + +```bash +# Set the version of the langchain package to verify +LANGCHAIN_VERSION=0.1.0 + +# Download package from PyPI +$ pip download --only-binary=:all: launchdarkly-server-sdk-ai-langchain==${LANGCHAIN_VERSION} + +# Download provenance from GitHub release into same directory $ curl --location -O \ - https://github.com/launchdarkly/python-server-sdk-ai/releases/download/${VERSION}/multiple.intoto.jsonl + https://github.com/launchdarkly/python-server-sdk-ai/releases/download/langchain-${LANGCHAIN_VERSION}/multiple.intoto.jsonl # Run slsa-verifier to verify provenance against package artifacts $ slsa-verifier verify-artifact \ --provenance-path multiple.intoto.jsonl \ --source-uri github.com/launchdarkly/python-server-sdk-ai \ -launchdarkly_server_sdk_ai-${VERSION}-py3-none-any.whl +launchdarkly_server_sdk_ai_langchain-${LANGCHAIN_VERSION}-py3-none-any.whl ``` -Below is a sample of expected output. +### Expected Output + +Below is a sample of expected output for successful verification: ``` Verified signature against tlog entry index 150910243 at URL: https://rekor.sigstore.dev/api/v1/log/entries/108e9186e8c5677ab3f14fc82cd3deb769e07ef812cadda623c08c77d4e51fc03124ee7542c470a1 Verified build using builder "https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v2.0.0" at commit 8e2d4094b4833d075e70dfce43bbc7176008c4a1 -Verifying artifact launchdarkly_server_sdk_ai-0.3.0-py3-none-any.whl: PASSED +Verifying artifact launchdarkly_server_sdk_ai-0.10.1-py3-none-any.whl: PASSED PASSED: SLSA verification passed ``` diff --git a/README.md b/README.md index 0411abe..8a983f3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # LaunchDarkly Server-side AI library for Python +> **Note:** This repository is a monorepo containing multiple packages. See the [Packages](#packages) section below. + ## LaunchDarkly overview [LaunchDarkly](https://www.launchdarkly.com) is a feature management platform that serves trillions of feature flags daily to help teams build better software, faster. [Get started](https://docs.launchdarkly.com/home/getting-started) using LaunchDarkly today! @@ -10,6 +12,15 @@ This version of the library has a minimum Python version of 3.9. +## Packages + +This repository contains the following packages: + +- **[`launchdarkly-server-sdk-ai`](./packages/core/)** - Core LaunchDarkly AI SDK +- **[`launchdarkly-server-sdk-ai-langchain`](./packages/langchain/)** - LangChain provider integration + +Refer to each package's README for specific installation and usage instructions. + ## Getting started Refer to the [SDK reference guide](https://docs.launchdarkly.com/sdk/ai/python) for instructions on getting started with using the SDK. @@ -32,7 +43,7 @@ LaunchDarkly uses the [SLSA framework](https://slsa.dev/spec/v1.0/about) (Supply - Roll out a new feature to a subset of your users (like a group of users who opt-in to a beta tester group), gathering feedback and bug reports from real-world use cases. - Gradually roll out a feature to an increasing percentage of users, and track the effect that the feature has on key metrics (for instance, how likely is a user to complete a purchase if they have feature A versus feature B?). - Turn off a feature that you realize is causing performance problems in production, without needing to re-deploy, or even restart the application with a changed configuration file. - - Grant access to certain features based on user attributes, like payment plan (eg: users on the ‘gold’ plan get access to more features than users in the ‘silver’ plan). Disable parts of your application to facilitate maintenance, without taking everything offline. + - Grant access to certain features based on user attributes, like payment plan (eg: users on the 'gold' plan get access to more features than users in the 'silver' plan). Disable parts of your application to facilitate maintenance, without taking everything offline. - LaunchDarkly provides feature flag SDKs for a wide variety of languages and technologies. Read [our documentation](https://docs.launchdarkly.com/sdk) for a complete list. - Explore LaunchDarkly - [launchdarkly.com](https://www.launchdarkly.com/ "LaunchDarkly Main Website") for more information diff --git a/ldai/__init__.py b/ldai/__init__.py deleted file mode 100644 index cb7e545..0000000 --- a/ldai/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.10.1" # x-release-please-version diff --git a/ldai/client.py b/ldai/client.py deleted file mode 100644 index a8bd888..0000000 --- a/ldai/client.py +++ /dev/null @@ -1,455 +0,0 @@ -from dataclasses import dataclass -from typing import Any, Dict, List, Literal, Optional, Tuple - -import chevron -from ldclient import Context -from ldclient.client import LDClient - -from ldai.tracker import LDAIConfigTracker - - -@dataclass -class LDMessage: - role: Literal['system', 'user', 'assistant'] - content: str - - def to_dict(self) -> dict: - """ - Render the given message as a dictionary object. - """ - return { - 'role': self.role, - 'content': self.content, - } - - -class ModelConfig: - """ - Configuration related to the model. - """ - - def __init__(self, name: str, parameters: Optional[Dict[str, Any]] = None, custom: Optional[Dict[str, Any]] = None): - """ - :param name: The name of the model. - :param parameters: Additional model-specific parameters. - :param custom: Additional customer provided data. - """ - self._name = name - self._parameters = parameters - self._custom = custom - - @property - def name(self) -> str: - """ - The name of the model. - """ - return self._name - - def get_parameter(self, key: str) -> Any: - """ - Retrieve model-specific parameters. - - Accessing a named, typed attribute (e.g. name) will result in the call - being delegated to the appropriate property. - """ - if key == 'name': - return self.name - - if self._parameters is None: - return None - - return self._parameters.get(key) - - def get_custom(self, key: str) -> Any: - """ - Retrieve customer provided data. - """ - if self._custom is None: - return None - - return self._custom.get(key) - - def to_dict(self) -> dict: - """ - Render the given model config as a dictionary object. - """ - return { - 'name': self._name, - 'parameters': self._parameters, - 'custom': self._custom, - } - - -class ProviderConfig: - """ - Configuration related to the provider. - """ - - def __init__(self, name: str): - self._name = name - - @property - def name(self) -> str: - """ - The name of the provider. - """ - return self._name - - def to_dict(self) -> dict: - """ - Render the given provider config as a dictionary object. - """ - return { - 'name': self._name, - } - - -@dataclass(frozen=True) -class AIConfig: - enabled: Optional[bool] = None - model: Optional[ModelConfig] = None - messages: Optional[List[LDMessage]] = None - provider: Optional[ProviderConfig] = None - - def to_dict(self) -> dict: - """ - Render the given default values as an AIConfig-compatible dictionary object. - """ - return { - '_ldMeta': { - 'enabled': self.enabled or False, - }, - 'model': self.model.to_dict() if self.model else None, - 'messages': [message.to_dict() for message in self.messages] if self.messages else None, - 'provider': self.provider.to_dict() if self.provider else None, - } - - -@dataclass(frozen=True) -class LDAIAgent: - """ - Represents an AI agent configuration with instructions and model settings. - - An agent is similar to an AIConfig but focuses on instructions rather than messages, - making it suitable for AI assistant/agent use cases. - """ - enabled: Optional[bool] = None - model: Optional[ModelConfig] = None - provider: Optional[ProviderConfig] = None - instructions: Optional[str] = None - tracker: Optional[LDAIConfigTracker] = None - - def to_dict(self) -> Dict[str, Any]: - """ - Render the given agent as a dictionary object. - """ - result: Dict[str, Any] = { - '_ldMeta': { - 'enabled': self.enabled or False, - }, - 'model': self.model.to_dict() if self.model else None, - 'provider': self.provider.to_dict() if self.provider else None, - } - if self.instructions is not None: - result['instructions'] = self.instructions - return result - - -@dataclass(frozen=True) -class LDAIAgentDefaults: - """ - Default values for AI agent configurations. - - Similar to LDAIAgent but without tracker and with optional enabled field, - used as fallback values when agent configurations are not available. - """ - enabled: Optional[bool] = None - model: Optional[ModelConfig] = None - provider: Optional[ProviderConfig] = None - instructions: Optional[str] = None - - def to_dict(self) -> Dict[str, Any]: - """ - Render the given agent defaults as a dictionary object. - """ - result: Dict[str, Any] = { - '_ldMeta': { - 'enabled': self.enabled or False, - }, - 'model': self.model.to_dict() if self.model else None, - 'provider': self.provider.to_dict() if self.provider else None, - } - if self.instructions is not None: - result['instructions'] = self.instructions - return result - - -@dataclass -class LDAIAgentConfig: - """ - Configuration for individual agent in batch requests. - - Combines agent key with its specific default configuration and variables. - """ - key: str - default_value: LDAIAgentDefaults - variables: Optional[Dict[str, Any]] = None - - -# Type alias for multiple agents -LDAIAgents = Dict[str, LDAIAgent] - - -class LDAIClient: - """The LaunchDarkly AI SDK client object.""" - - def __init__(self, client: LDClient): - self._client = client - - def config( - self, - key: str, - context: Context, - default_value: AIConfig, - variables: Optional[Dict[str, Any]] = None, - ) -> Tuple[AIConfig, LDAIConfigTracker]: - """ - Get the value of a model configuration. - - :param key: The key of the model configuration. - :param context: The context to evaluate the model configuration in. - :param default_value: The default value of the model configuration. - :param variables: Additional variables for the model configuration. - :return: The value of the model configuration along with a tracker used for gathering metrics. - """ - self._client.track('$ld:ai:config:function:single', context, key, 1) - - model, provider, messages, instructions, tracker, enabled = self.__evaluate(key, context, default_value.to_dict(), variables) - - config = AIConfig( - enabled=bool(enabled), - model=model, - messages=messages, - provider=provider, - ) - - return config, tracker - - def agent( - self, - config: LDAIAgentConfig, - context: Context, - ) -> LDAIAgent: - """ - Retrieve a single AI Config agent. - - This method retrieves a single agent configuration with instructions - dynamically interpolated using the provided variables and context data. - - Example:: - - agent = client.agent(LDAIAgentConfig( - key='research_agent', - default_value=LDAIAgentDefaults( - enabled=True, - model=ModelConfig('gpt-4'), - instructions="You are a research assistant specializing in {{topic}}." - ), - variables={'topic': 'climate change'} - ), context) - - if agent.enabled: - research_result = agent.instructions # Interpolated instructions - agent.tracker.track_success() - - :param config: The agent configuration to use. - :param context: The context to evaluate the agent configuration in. - :return: Configured LDAIAgent instance. - """ - # Track single agent usage - self._client.track( - "$ld:ai:agent:function:single", - context, - config.key, - 1 - ) - - return self.__evaluate_agent(config.key, context, config.default_value, config.variables) - - def agents( - self, - agent_configs: List[LDAIAgentConfig], - context: Context, - ) -> LDAIAgents: - """ - Retrieve multiple AI agent configurations. - - This method allows you to retrieve multiple agent configurations in a single call, - with each agent having its own default configuration and variables for instruction - interpolation. - - Example:: - - agents = client.agents([ - LDAIAgentConfig( - key='research_agent', - default_value=LDAIAgentDefaults( - enabled=True, - instructions='You are a research assistant.' - ), - variables={'topic': 'climate change'} - ), - LDAIAgentConfig( - key='writing_agent', - default_value=LDAIAgentDefaults( - enabled=True, - instructions='You are a writing assistant.' - ), - variables={'style': 'academic'} - ) - ], context) - - research_result = agents["research_agent"].instructions - agents["research_agent"].tracker.track_success() - - :param agent_configs: List of agent configurations to retrieve. - :param context: The context to evaluate the agent configurations in. - :return: Dictionary mapping agent keys to their LDAIAgent configurations. - """ - # Track multiple agents usage - agent_count = len(agent_configs) - self._client.track( - "$ld:ai:agent:function:multiple", - context, - agent_count, - agent_count - ) - - result: LDAIAgents = {} - - for config in agent_configs: - agent = self.__evaluate_agent( - config.key, - context, - config.default_value, - config.variables - ) - result[config.key] = agent - - return result - - def __evaluate( - self, - key: str, - context: Context, - default_dict: Dict[str, Any], - variables: Optional[Dict[str, Any]] = None, - ) -> Tuple[Optional[ModelConfig], Optional[ProviderConfig], Optional[List[LDMessage]], Optional[str], LDAIConfigTracker, bool]: - """ - Internal method to evaluate a configuration and extract components. - - :param key: The configuration key. - :param context: The evaluation context. - :param default_dict: Default configuration as dictionary. - :param variables: Variables for interpolation. - :return: Tuple of (model, provider, messages, instructions, tracker, enabled). - """ - variation = self._client.variation(key, context, default_dict) - - all_variables = {} - if variables: - all_variables.update(variables) - all_variables['ldctx'] = context.to_dict() - - # Extract messages - messages = None - if 'messages' in variation and isinstance(variation['messages'], list) and all( - isinstance(entry, dict) for entry in variation['messages'] - ): - messages = [ - LDMessage( - role=entry['role'], - content=self.__interpolate_template( - entry['content'], all_variables - ), - ) - for entry in variation['messages'] - ] - - # Extract instructions - instructions = None - if 'instructions' in variation and isinstance(variation['instructions'], str): - instructions = self.__interpolate_template(variation['instructions'], all_variables) - - # Extract provider config - provider_config = None - if 'provider' in variation and isinstance(variation['provider'], dict): - provider = variation['provider'] - provider_config = ProviderConfig(provider.get('name', '')) - - # Extract model config - model = None - if 'model' in variation and isinstance(variation['model'], dict): - parameters = variation['model'].get('parameters', None) - custom = variation['model'].get('custom', None) - model = ModelConfig( - name=variation['model']['name'], - parameters=parameters, - custom=custom - ) - - # Create tracker - tracker = LDAIConfigTracker( - self._client, - variation.get('_ldMeta', {}).get('variationKey', ''), - key, - int(variation.get('_ldMeta', {}).get('version', 1)), - model.name if model else '', - provider_config.name if provider_config else '', - context, - ) - - enabled = variation.get('_ldMeta', {}).get('enabled', False) - - return model, provider_config, messages, instructions, tracker, enabled - - def __evaluate_agent( - self, - key: str, - context: Context, - default_value: LDAIAgentDefaults, - variables: Optional[Dict[str, Any]] = None, - ) -> LDAIAgent: - """ - Internal method to evaluate an agent configuration. - - :param key: The agent configuration key. - :param context: The evaluation context. - :param default_value: Default agent values. - :param variables: Variables for interpolation. - :return: Configured LDAIAgent instance. - """ - model, provider, messages, instructions, tracker, enabled = self.__evaluate( - key, context, default_value.to_dict(), variables - ) - - # For agents, prioritize instructions over messages - final_instructions = instructions if instructions is not None else default_value.instructions - - return LDAIAgent( - enabled=bool(enabled) if enabled is not None else default_value.enabled, - model=model or default_value.model, - provider=provider or default_value.provider, - instructions=final_instructions, - tracker=tracker, - ) - - def __interpolate_template(self, template: str, variables: Dict[str, Any]) -> str: - """ - Interpolate the template with the given variables using Mustache format. - - :param template: The template string. - :param variables: The variables to interpolate into the template. - :return: The interpolated string. - """ - return chevron.render(template, variables) diff --git a/ldai/testing/__init__.py b/ldai/testing/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/packages/core/LICENSE.txt b/packages/core/LICENSE.txt new file mode 100644 index 0000000..50add35 --- /dev/null +++ b/packages/core/LICENSE.txt @@ -0,0 +1,13 @@ +Copyright 2024 Catamorphic, Co. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/core/README.md b/packages/core/README.md new file mode 100644 index 0000000..0411abe --- /dev/null +++ b/packages/core/README.md @@ -0,0 +1,41 @@ +# LaunchDarkly Server-side AI library for Python + +## LaunchDarkly overview + +[LaunchDarkly](https://www.launchdarkly.com) is a feature management platform that serves trillions of feature flags daily to help teams build better software, faster. [Get started](https://docs.launchdarkly.com/home/getting-started) using LaunchDarkly today! + +[![Twitter Follow](https://img.shields.io/twitter/follow/launchdarkly.svg?style=social&label=Follow&maxAge=2592000)](https://twitter.com/intent/follow?screen_name=launchdarkly) + +## Supported Python versions + +This version of the library has a minimum Python version of 3.9. + +## Getting started + +Refer to the [SDK reference guide](https://docs.launchdarkly.com/sdk/ai/python) for instructions on getting started with using the SDK. + +## Learn more + +Read our [documentation](http://docs.launchdarkly.com) for in-depth instructions on configuring and using LaunchDarkly. You can also head straight to the [reference guide for the python SDK](http://docs.launchdarkly.com/docs/python-sdk-ai-reference). + +## Contributing + +We encourage pull requests and other contributions from the community. Check out our [contributing guidelines](CONTRIBUTING.md) for instructions on how to contribute to this library. + +## Verifying library build provenance with the SLSA framework + +LaunchDarkly uses the [SLSA framework](https://slsa.dev/spec/v1.0/about) (Supply-chain Levels for Software Artifacts) to help developers make their supply chain more secure by ensuring the authenticity and build integrity of our published library packages. To learn more, see the [provenance guide](PROVENANCE.md). + +## About LaunchDarkly + +- LaunchDarkly is a continuous delivery platform that provides feature flags as a service and allows developers to iterate quickly and safely. We allow you to easily flag your features and manage them from the LaunchDarkly dashboard. With LaunchDarkly, you can: + - Roll out a new feature to a subset of your users (like a group of users who opt-in to a beta tester group), gathering feedback and bug reports from real-world use cases. + - Gradually roll out a feature to an increasing percentage of users, and track the effect that the feature has on key metrics (for instance, how likely is a user to complete a purchase if they have feature A versus feature B?). + - Turn off a feature that you realize is causing performance problems in production, without needing to re-deploy, or even restart the application with a changed configuration file. + - Grant access to certain features based on user attributes, like payment plan (eg: users on the ‘gold’ plan get access to more features than users in the ‘silver’ plan). Disable parts of your application to facilitate maintenance, without taking everything offline. +- LaunchDarkly provides feature flag SDKs for a wide variety of languages and technologies. Read [our documentation](https://docs.launchdarkly.com/sdk) for a complete list. +- Explore LaunchDarkly + - [launchdarkly.com](https://www.launchdarkly.com/ "LaunchDarkly Main Website") for more information + - [docs.launchdarkly.com](https://docs.launchdarkly.com/ "LaunchDarkly Documentation") for our documentation and SDK reference guides + - [apidocs.launchdarkly.com](https://apidocs.launchdarkly.com/ "LaunchDarkly API Documentation") for our API documentation + - [blog.launchdarkly.com](https://blog.launchdarkly.com/ "LaunchDarkly Blog Documentation") for the latest product updates diff --git a/packages/core/ldai/__init__.py b/packages/core/ldai/__init__.py new file mode 100644 index 0000000..2e40fb6 --- /dev/null +++ b/packages/core/ldai/__init__.py @@ -0,0 +1,49 @@ +__version__ = "0.10.1" # x-release-please-version + +# Extend __path__ to support namespace packages at the ldai level +# This allows provider packages (like launchdarkly-server-sdk-ai-langchain) +# to extend ldai.providers.* even though ldai itself has an __init__.py +import sys +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) + +# Export chat +from ldai.chat import TrackedChat +# Export main client +from ldai.client import LDAIClient +# Export judge +from ldai.judge import AIJudge +# Export models for convenience +from ldai.models import ( # Deprecated aliases for backward compatibility + AIAgentConfig, AIAgentConfigDefault, AIAgentConfigRequest, AIAgents, + AICompletionConfig, AICompletionConfigDefault, AIConfig, AIJudgeConfig, + AIJudgeConfigDefault, JudgeConfiguration, LDAIAgent, LDAIAgentConfig, + LDAIAgentDefaults, LDMessage, ModelConfig, ProviderConfig) +# Export judge types +from ldai.providers.types import EvalScore, JudgeResponse + +__all__ = [ + 'LDAIClient', + 'AIAgentConfig', + 'AIAgentConfigDefault', + 'AIAgentConfigRequest', + 'AIAgents', + 'AICompletionConfig', + 'AICompletionConfigDefault', + 'AIJudgeConfig', + 'AIJudgeConfigDefault', + 'AIJudge', + 'TrackedChat', + 'EvalScore', + 'JudgeConfiguration', + 'JudgeResponse', + 'LDMessage', + 'ModelConfig', + 'ProviderConfig', + # Deprecated exports + 'AIConfig', + 'LDAIAgent', + 'LDAIAgentConfig', + 'LDAIAgentDefaults', +] diff --git a/packages/core/ldai/chat/__init__.py b/packages/core/ldai/chat/__init__.py new file mode 100644 index 0000000..265a1b3 --- /dev/null +++ b/packages/core/ldai/chat/__init__.py @@ -0,0 +1,5 @@ +"""Chat module for LaunchDarkly AI SDK.""" + +from ldai.chat.tracked_chat import TrackedChat + +__all__ = ['TrackedChat'] diff --git a/packages/core/ldai/chat/tracked_chat.py b/packages/core/ldai/chat/tracked_chat.py new file mode 100644 index 0000000..e7bd8f3 --- /dev/null +++ b/packages/core/ldai/chat/tracked_chat.py @@ -0,0 +1,183 @@ +"""TrackedChat implementation for managing AI chat conversations.""" + +import asyncio +import logging +from typing import Dict, List, Optional + +from ldai.judge import AIJudge +from ldai.models import AICompletionConfig, LDMessage +from ldai.providers.ai_provider import AIProvider +from ldai.providers.types import ChatResponse, JudgeResponse +from ldai.tracker import LDAIConfigTracker + + +class TrackedChat: + """ + Concrete implementation of TrackedChat that provides chat functionality + by delegating to an AIProvider implementation. + + This class handles conversation management and tracking, while delegating + the actual model invocation to the provider. + """ + + def __init__( + self, + ai_config: AICompletionConfig, + tracker: LDAIConfigTracker, + provider: AIProvider, + judges: Optional[Dict[str, AIJudge]] = None, + ): + """ + Initialize the TrackedChat. + + :param ai_config: The completion AI configuration + :param tracker: The tracker for the completion configuration + :param provider: The AI provider to use for chat + :param judges: Optional dictionary of judge instances keyed by their configuration keys + """ + self._ai_config = ai_config + self._tracker = tracker + self._provider = provider + self._judges = judges or {} + self._logger = logging.getLogger(f'{__name__}.{self.__class__.__name__}') + self._messages: List[LDMessage] = [] + + async def invoke(self, prompt: str) -> ChatResponse: + """ + Invoke the chat model with a prompt string. + + This method handles conversation management and tracking, delegating to the provider's invoke_model method. + + :param prompt: The user prompt to send to the chat model + :return: ChatResponse containing the model's response and metrics + """ + # Convert prompt string to LDMessage with role 'user' and add to conversation history + user_message: LDMessage = LDMessage(role='user', content=prompt) + self._messages.append(user_message) + + # Prepend config messages to conversation history for model invocation + config_messages = self._ai_config.messages or [] + all_messages = config_messages + self._messages + + # Delegate to provider-specific implementation with tracking + response = await self._tracker.track_metrics_of( + lambda result: result.metrics, + lambda: self._provider.invoke_model(all_messages), + ) + + # Start judge evaluations as async tasks (don't await them) + judge_config = self._ai_config.judge_configuration + if judge_config and judge_config.judges and len(judge_config.judges) > 0: + evaluation_tasks = self._start_judge_evaluations(self._messages, response) + response.evaluations = evaluation_tasks + + # Add the response message to conversation history + self._messages.append(response.message) + return response + + def _start_judge_evaluations( + self, + messages: List[LDMessage], + response: ChatResponse, + ) -> List[asyncio.Task[Optional[JudgeResponse]]]: + """ + Start judge evaluations as async tasks without awaiting them. + + Returns a list of async tasks that can be awaited later. + + :param messages: Array of messages representing the conversation history + :param response: The AI response to be evaluated + :return: List of async tasks that will return judge evaluation results + """ + if not self._ai_config.judge_configuration or not self._ai_config.judge_configuration.judges: + return [] + + judge_configs = self._ai_config.judge_configuration.judges + + # Start all judge evaluations as tasks + async def evaluate_judge(judge_config): + judge = self._judges.get(judge_config.key) + if not judge: + self._logger.warning( + f"Judge configuration is not enabled: {judge_config.key}", + ) + return None + + eval_result = await judge.evaluate_messages( + messages, response, judge_config.sampling_rate + ) + + if eval_result and eval_result.success: + self._tracker.track_eval_scores(eval_result.evals) + + return eval_result + + # Create tasks for each judge evaluation + tasks = [ + asyncio.create_task(evaluate_judge(judge_config)) + for judge_config in judge_configs + ] + + return tasks + + def get_config(self) -> AICompletionConfig: + """ + Get the underlying AI configuration used to initialize this TrackedChat. + + :return: The AI completion configuration + """ + return self._ai_config + + def get_tracker(self) -> LDAIConfigTracker: + """ + Get the underlying AI configuration tracker used to initialize this TrackedChat. + + :return: The tracker instance + """ + return self._tracker + + def get_provider(self) -> AIProvider: + """ + Get the underlying AI provider instance. + + This provides direct access to the provider for advanced use cases. + + :return: The AI provider instance + """ + return self._provider + + def get_judges(self) -> Dict[str, AIJudge]: + """ + Get the judges associated with this TrackedChat. + + Returns a dictionary of judge instances keyed by their configuration keys. + + :return: Dictionary of judge instances + """ + return self._judges + + def append_messages(self, messages: List[LDMessage]) -> None: + """ + Append messages to the conversation history. + + Adds messages to the conversation history without invoking the model, + which is useful for managing multi-turn conversations or injecting context. + + :param messages: Array of messages to append to the conversation history + """ + self._messages.extend(messages) + + def get_messages(self, include_config_messages: bool = False) -> List[LDMessage]: + """ + Get all messages in the conversation history. + + :param include_config_messages: Whether to include the config messages from the AIConfig. + Defaults to False. + :return: Array of messages. When include_config_messages is True, returns both config + messages and conversation history with config messages prepended. When False, + returns only the conversation history messages. + """ + if include_config_messages: + config_messages = self._ai_config.messages or [] + return config_messages + self._messages + return list(self._messages) diff --git a/packages/core/ldai/client.py b/packages/core/ldai/client.py new file mode 100644 index 0000000..2b314cf --- /dev/null +++ b/packages/core/ldai/client.py @@ -0,0 +1,573 @@ +import asyncio +import logging +from typing import Any, Dict, List, Optional, Tuple + +import chevron +from ldclient import Context +from ldclient.client import LDClient + +from ldai.chat import TrackedChat +from ldai.judge import AIJudge +from ldai.models import (AIAgentConfig, AIAgentConfigDefault, + AIAgentConfigRequest, AIAgents, AICompletionConfig, + AICompletionConfigDefault, AIJudgeConfig, + AIJudgeConfigDefault, JudgeConfiguration, LDMessage, + ModelConfig, ProviderConfig) +from ldai.providers.ai_provider_factory import (AIProviderFactory, + SupportedAIProvider) +from ldai.tracker import LDAIConfigTracker + + +class LDAIClient: + """The LaunchDarkly AI SDK client object.""" + + def __init__(self, client: LDClient): + self._client = client + self._logger = logging.getLogger('ldclient.ai') + + def completion_config( + self, + key: str, + context: Context, + default_value: AICompletionConfigDefault, + variables: Optional[Dict[str, Any]] = None, + ) -> AICompletionConfig: + """ + Get the value of a completion configuration. + + :param key: The key of the completion configuration. + :param context: The context to evaluate the completion configuration in. + :param default_value: The default value of the completion configuration. + :param variables: Additional variables for the completion configuration. + :return: The completion configuration with a tracker used for gathering metrics. + """ + self._client.track('$ld:ai:config:function:single', context, key, 1) + + model, provider, messages, instructions, tracker, enabled, judge_configuration = self.__evaluate( + key, context, default_value.to_dict(), variables + ) + + config = AICompletionConfig( + enabled=enabled, + model=model, + messages=messages, + provider=provider, + tracker=tracker, + judge_configuration=judge_configuration, + ) + + return config + + def config( + self, + key: str, + context: Context, + default_value: AICompletionConfigDefault, + variables: Optional[Dict[str, Any]] = None, + ) -> AICompletionConfig: + """ + Get the value of a model configuration. + + .. deprecated:: Use :meth:`completion_config` instead. This method will be removed in a future version. + + :param key: The key of the model configuration. + :param context: The context to evaluate the model configuration in. + :param default_value: The default value of the model configuration. + :param variables: Additional variables for the model configuration. + :return: The value of the model configuration along with a tracker used for gathering metrics. + """ + return self.completion_config(key, context, default_value, variables) + + def judge_config( + self, + key: str, + context: Context, + default_value: AIJudgeConfigDefault, + variables: Optional[Dict[str, Any]] = None, + ) -> AIJudgeConfig: + """ + Get the value of a judge configuration. + + :param key: The key of the judge configuration. + :param context: The context to evaluate the judge configuration in. + :param default_value: The default value of the judge configuration. + :param variables: Additional variables for the judge configuration. + :return: The judge configuration with a tracker used for gathering metrics. + """ + self._client.track('$ld:ai:judge:function:single', context, key, 1) + + model, provider, messages, instructions, tracker, enabled, judge_configuration = self.__evaluate( + key, context, default_value.to_dict(), variables + ) + + # Extract evaluation_metric_keys from the variation + variation = self._client.variation(key, context, default_value.to_dict()) + evaluation_metric_keys = variation.get('evaluationMetricKeys', default_value.evaluation_metric_keys or []) + + config = AIJudgeConfig( + enabled=enabled, + evaluation_metric_keys=evaluation_metric_keys, + model=model, + messages=messages, + provider=provider, + tracker=tracker, + ) + + return config + + async def create_judge( + self, + key: str, + context: Context, + default_value: AIJudgeConfigDefault, + variables: Optional[Dict[str, Any]] = None, + default_ai_provider: Optional[SupportedAIProvider] = None, + ) -> Optional[AIJudge]: + """ + Creates and returns a new Judge instance for AI evaluation. + + :param key: The key identifying the AI judge configuration to use + :param context: Standard Context used when evaluating flags + :param default_value: A default value representing a standard AI config result + :param variables: Dictionary of values for instruction interpolation. + The variables `message_history` and `response_to_evaluate` are reserved for the judge and will be ignored. + :param default_ai_provider: Optional default AI provider to use. + :return: Judge instance or None if disabled/unsupported + + Example:: + + judge = client.create_judge( + "relevance-judge", + context, + AIJudgeConfigDefault( + enabled=True, + model=ModelConfig("gpt-4"), + provider=ProviderConfig("openai"), + evaluation_metric_keys=['$ld:ai:judge:relevance'], + messages=[LDMessage(role='system', content='You are a relevance judge.')] + ), + variables={'metric': "relevance"} + ) + + if judge: + result = await judge.evaluate("User question", "AI response") + if result and result.evals: + relevance_eval = result.evals.get('$ld:ai:judge:relevance') + if relevance_eval: + print('Relevance score:', relevance_eval.score) + """ + self._client.track('$ld:ai:judge:function:createJudge', context, key, 1) + + try: + # Overwrite reserved variables to ensure they remain as placeholders for judge evaluation + extended_variables = dict(variables) if variables else {} + + # Warn if reserved variables are provided + if variables: + if 'message_history' in variables: + self._logger.warning( + 'Variable "message_history" is reserved for judge evaluation and will be overwritten' + ) + if 'response_to_evaluate' in variables: + self._logger.warning( + 'Variable "response_to_evaluate" is reserved for judge evaluation and will be overwritten' + ) + + extended_variables['message_history'] = '{{message_history}}' + extended_variables['response_to_evaluate'] = '{{response_to_evaluate}}' + + judge_config = self.judge_config(key, context, default_value, extended_variables) + + if not judge_config.enabled or not judge_config.tracker: + return None + + # Create AI provider for the judge + provider = await AIProviderFactory.create(judge_config, default_ai_provider) + if not provider: + return None + + return AIJudge(judge_config, judge_config.tracker, provider) + except Exception as error: + self._logger.error(f'Failed to create judge: {error}') + return None + + async def _initialize_judges( + self, + judge_configs: List[JudgeConfiguration.Judge], + context: Context, + variables: Optional[Dict[str, Any]] = None, + default_ai_provider: Optional[SupportedAIProvider] = None, + ) -> Dict[str, AIJudge]: + """ + Initialize judges from judge configurations. + + :param judge_configs: List of judge configurations + :param context: Standard Context used when evaluating flags + :param variables: Dictionary of values for instruction interpolation + :param default_ai_provider: Optional default AI provider to use + :return: Dictionary of judge instances keyed by their configuration keys + """ + judges: Dict[str, AIJudge] = {} + + async def create_judge_for_config(judge_key: str): + judge = await self.create_judge( + judge_key, + context, + AIJudgeConfigDefault(enabled=False), + variables, + default_ai_provider, + ) + return judge_key, judge + + judge_promises = [ + create_judge_for_config(judge_config.key) + for judge_config in judge_configs + ] + + results = await asyncio.gather(*judge_promises, return_exceptions=True) + + for result in results: + if isinstance(result, Exception): + continue + judge_key, judge = result # type: ignore[misc] + if judge: + judges[judge_key] = judge + + return judges + + async def create_chat( + self, + key: str, + context: Context, + default_value: AICompletionConfigDefault, + variables: Optional[Dict[str, Any]] = None, + default_ai_provider: Optional[SupportedAIProvider] = None, + ) -> Optional[TrackedChat]: + """ + Creates and returns a new TrackedChat instance for AI chat conversations. + + :param key: The key identifying the AI completion configuration to use + :param context: Standard Context used when evaluating flags + :param default_value: A default value representing a standard AI config result + :param variables: Dictionary of values for instruction interpolation + :param default_ai_provider: Optional default AI provider to use + :return: TrackedChat instance or None if disabled/unsupported + + Example:: + + chat = await client.create_chat( + "customer-support-chat", + context, + AICompletionConfigDefault( + enabled=True, + model=ModelConfig("gpt-4"), + provider=ProviderConfig("openai"), + messages=[LDMessage(role='system', content='You are a helpful assistant.')] + ), + variables={'customerName': 'John'} + ) + + if chat: + response = await chat.invoke("I need help with my order") + print(response.message.content) + + # Access conversation history + messages = chat.get_messages() + print(f"Conversation has {len(messages)} messages") + """ + self._client.track('$ld:ai:config:function:createChat', context, key, 1) + if self._logger: + self._logger.debug(f"Creating chat for key: {key}") + config = self.completion_config(key, context, default_value, variables) + + if not config.enabled or not config.tracker: + return None + + provider = await AIProviderFactory.create(config, default_ai_provider) + if not provider: + return None + + judges = {} + if config.judge_configuration and config.judge_configuration.judges: + judges = await self._initialize_judges( + config.judge_configuration.judges, + context, + variables, + default_ai_provider, + ) + + return TrackedChat(config, config.tracker, provider, judges) + + def agent_config( + self, + key: str, + context: Context, + default_value: AIAgentConfigDefault, + variables: Optional[Dict[str, Any]] = None, + ) -> AIAgentConfig: + """ + Retrieve a single AI Config agent. + + This method retrieves a single agent configuration with instructions + dynamically interpolated using the provided variables and context data. + + Example:: + + agent = client.agent_config( + 'research_agent', + context, + AIAgentConfigDefault( + enabled=True, + model=ModelConfig('gpt-4'), + instructions="You are a research assistant specializing in {{topic}}." + ), + variables={'topic': 'climate change'} + ) + + if agent.enabled: + research_result = agent.instructions # Interpolated instructions + agent.tracker.track_success() + + :param key: The agent configuration key. + :param context: The context to evaluate the agent configuration in. + :param default_value: Default agent values. + :param variables: Variables for interpolation. + :return: Configured AIAgentConfig instance. + """ + # Track single agent usage + self._client.track( + "$ld:ai:agent:function:single", + context, + key, + 1 + ) + + return self.__evaluate_agent(key, context, default_value, variables) + + def agent( + self, + config: AIAgentConfigRequest, + context: Context, + ) -> AIAgentConfig: + """ + Retrieve a single AI Config agent. + + .. deprecated:: Use :meth:`agent_config` instead. This method will be removed in a future version. + + :param config: The agent configuration to use. + :param context: The context to evaluate the agent configuration in. + :return: Configured AIAgentConfig instance. + """ + return self.agent_config(config.key, context, config.default_value, config.variables) + + def agent_configs( + self, + agent_configs: List[AIAgentConfigRequest], + context: Context, + ) -> AIAgents: + """ + Retrieve multiple AI agent configurations. + + This method allows you to retrieve multiple agent configurations in a single call, + with each agent having its own default configuration and variables for instruction + interpolation. + + Example:: + + agents = client.agent_configs([ + AIAgentConfigRequest( + key='research_agent', + default_value=AIAgentConfigDefault( + enabled=True, + instructions='You are a research assistant.' + ), + variables={'topic': 'climate change'} + ), + AIAgentConfigRequest( + key='writing_agent', + default_value=AIAgentConfigDefault( + enabled=True, + instructions='You are a writing assistant.' + ), + variables={'style': 'academic'} + ) + ], context) + + research_result = agents["research_agent"].instructions + agents["research_agent"].tracker.track_success() + + :param agent_configs: List of agent configurations to retrieve. + :param context: The context to evaluate the agent configurations in. + :return: Dictionary mapping agent keys to their AIAgentConfig configurations. + """ + # Track multiple agents usage + agent_count = len(agent_configs) + config_keys = [config.key for config in agent_configs] + self._client.track( + "$ld:ai:agent:function:multiple", + context, + {"configKeys": config_keys}, + agent_count + ) + + result: AIAgents = {} + + for config in agent_configs: + agent = self.__evaluate_agent( + config.key, + context, + config.default_value, + config.variables + ) + result[config.key] = agent + + return result + + def agents( + self, + agent_configs: List[AIAgentConfigRequest], + context: Context, + ) -> AIAgents: + """ + Retrieve multiple AI agent configurations. + + .. deprecated:: Use :meth:`agent_configs` instead. This method will be removed in a future version. + + :param agent_configs: List of agent configurations to retrieve. + :param context: The context to evaluate the agent configurations in. + :return: Dictionary mapping agent keys to their AIAgentConfig configurations. + """ + return self.agent_configs(agent_configs, context) + + def __evaluate( + self, + key: str, + context: Context, + default_dict: Dict[str, Any], + variables: Optional[Dict[str, Any]] = None, + ) -> Tuple[Optional[ModelConfig], Optional[ProviderConfig], Optional[List[LDMessage]], Optional[str], LDAIConfigTracker, bool, Optional[Any]]: + """ + Internal method to evaluate a configuration and extract components. + + :param key: The configuration key. + :param context: The evaluation context. + :param default_dict: Default configuration as dictionary. + :param variables: Variables for interpolation. + :return: Tuple of (model, provider, messages, instructions, tracker, enabled). + """ + variation = self._client.variation(key, context, default_dict) + + all_variables = {} + if variables: + all_variables.update(variables) + all_variables['ldctx'] = context.to_dict() + + # Extract messages + messages = None + if 'messages' in variation and isinstance(variation['messages'], list) and all( + isinstance(entry, dict) for entry in variation['messages'] + ): + messages = [ + LDMessage( + role=entry['role'], + content=self.__interpolate_template( + entry['content'], all_variables + ), + ) + for entry in variation['messages'] + ] + + # Extract instructions + instructions = None + if 'instructions' in variation and isinstance(variation['instructions'], str): + instructions = self.__interpolate_template(variation['instructions'], all_variables) + + # Extract provider config + provider_config = None + if 'provider' in variation and isinstance(variation['provider'], dict): + provider = variation['provider'] + provider_config = ProviderConfig(provider.get('name', '')) + + # Extract model config + model = None + if 'model' in variation and isinstance(variation['model'], dict): + parameters = variation['model'].get('parameters', None) + custom = variation['model'].get('custom', None) + model = ModelConfig( + name=variation['model']['name'], + parameters=parameters, + custom=custom + ) + + # Create tracker + tracker = LDAIConfigTracker( + self._client, + variation.get('_ldMeta', {}).get('variationKey', ''), + key, + int(variation.get('_ldMeta', {}).get('version', 1)), + model.name if model else '', + provider_config.name if provider_config else '', + context, + ) + + enabled = variation.get('_ldMeta', {}).get('enabled', False) + + # Extract judge configuration + judge_configuration = None + if 'judgeConfiguration' in variation and isinstance(variation['judgeConfiguration'], dict): + judge_config = variation['judgeConfiguration'] + if 'judges' in judge_config and isinstance(judge_config['judges'], list): + judges = [ + JudgeConfiguration.Judge( + key=judge['key'], + sampling_rate=judge['samplingRate'] + ) + for judge in judge_config['judges'] + if isinstance(judge, dict) and 'key' in judge and 'samplingRate' in judge + ] + if judges: + judge_configuration = JudgeConfiguration(judges=judges) + + return model, provider_config, messages, instructions, tracker, enabled, judge_configuration + + def __evaluate_agent( + self, + key: str, + context: Context, + default_value: AIAgentConfigDefault, + variables: Optional[Dict[str, Any]] = None, + ) -> AIAgentConfig: + """ + Internal method to evaluate an agent configuration. + + :param key: The agent configuration key. + :param context: The evaluation context. + :param default_value: Default agent values. + :param variables: Variables for interpolation. + :return: Configured AIAgentConfig instance. + """ + model, provider, messages, instructions, tracker, enabled, judge_configuration = self.__evaluate( + key, context, default_value.to_dict(), variables + ) + + # For agents, prioritize instructions over messages + final_instructions = instructions if instructions is not None else default_value.instructions + + return AIAgentConfig( + enabled=enabled, + model=model or default_value.model, + provider=provider or default_value.provider, + instructions=final_instructions, + tracker=tracker, + judge_configuration=judge_configuration or default_value.judge_configuration, + ) + + def __interpolate_template(self, template: str, variables: Dict[str, Any]) -> str: + """ + Interpolate the template with the given variables using Mustache format. + + :param template: The template string. + :param variables: The variables to interpolate into the template. + :return: The interpolated string. + """ + return chevron.render(template, variables) diff --git a/packages/core/ldai/judge/__init__.py b/packages/core/ldai/judge/__init__.py new file mode 100644 index 0000000..0660d0e --- /dev/null +++ b/packages/core/ldai/judge/__init__.py @@ -0,0 +1,5 @@ +"""Judge module for LaunchDarkly AI SDK.""" + +from ldai.judge.ai_judge import AIJudge + +__all__ = ['AIJudge'] diff --git a/packages/core/ldai/judge/ai_judge.py b/packages/core/ldai/judge/ai_judge.py new file mode 100644 index 0000000..7b1f3ea --- /dev/null +++ b/packages/core/ldai/judge/ai_judge.py @@ -0,0 +1,220 @@ +"""Judge implementation for AI evaluation.""" + +import logging +import random +from typing import Any, Dict, List, Optional + +import chevron + +from ldai.judge.evaluation_schema_builder import EvaluationSchemaBuilder +from ldai.models import AIJudgeConfig, LDMessage +from ldai.providers.ai_provider import AIProvider +from ldai.providers.types import (ChatResponse, EvalScore, JudgeResponse, + StructuredResponse) +from ldai.tracker import LDAIConfigTracker + + +class AIJudge: + """ + Judge implementation that handles evaluation functionality and conversation management. + + According to the AIEval spec, judges are AI Configs with mode: "judge" that evaluate + other AI Configs using structured output. + """ + + def __init__( + self, + ai_config: AIJudgeConfig, + ai_config_tracker: LDAIConfigTracker, + ai_provider: AIProvider, + ): + """ + Initialize the Judge. + + :param ai_config: The judge AI configuration + :param ai_config_tracker: The tracker for the judge configuration + :param ai_provider: The AI provider to use for evaluation + """ + self._ai_config = ai_config + self._ai_config_tracker = ai_config_tracker + self._ai_provider = ai_provider + self._logger = logging.getLogger(f'{__name__}.{self.__class__.__name__}') + self._evaluation_response_structure = EvaluationSchemaBuilder.build( + ai_config.evaluation_metric_keys + ) + + async def evaluate( + self, + input_text: str, + output_text: str, + sampling_rate: float = 1.0, + ) -> Optional[JudgeResponse]: + """ + Evaluates an AI response using the judge's configuration. + + :param input_text: The input prompt or question that was provided to the AI + :param output_text: The AI-generated response to be evaluated + :param sampling_rate: Sampling rate (0-1) to determine if evaluation should be processed (defaults to 1) + :return: Evaluation results or None if not sampled + """ + try: + if not self._ai_config.evaluation_metric_keys or len(self._ai_config.evaluation_metric_keys) == 0: + self._logger.warning( + 'Judge configuration is missing required evaluationMetricKeys' + ) + return None + + if not self._ai_config.messages: + self._logger.warning('Judge configuration must include messages') + return None + + if random.random() > sampling_rate: + self._logger.debug(f'Judge evaluation skipped due to sampling rate: {sampling_rate}') + return None + + messages = self._construct_evaluation_messages(input_text, output_text) + + # Track metrics of the structured model invocation + response = await self._ai_config_tracker.track_metrics_of( + lambda result: result.metrics, + lambda: self._ai_provider.invoke_structured_model(messages, self._evaluation_response_structure) + ) + + success = response.metrics.success + + evals = self._parse_evaluation_response(response.data) + + if len(evals) != len(self._ai_config.evaluation_metric_keys): + self._logger.warning('Judge evaluation did not return all evaluations') + success = False + + return JudgeResponse( + evals=evals, + success=success, + ) + except Exception as error: + self._logger.error(f'Judge evaluation failed: {error}') + return JudgeResponse( + evals={}, + success=False, + error=str(error), + ) + + async def evaluate_messages( + self, + messages: List[LDMessage], + response: ChatResponse, + sampling_ratio: float = 1.0, + ) -> Optional[JudgeResponse]: + """ + Evaluates an AI response from chat messages and response. + + :param messages: Array of messages representing the conversation history + :param response: The AI response to be evaluated + :param sampling_ratio: Sampling ratio (0-1) to determine if evaluation should be processed (defaults to 1) + :return: Evaluation results or None if not sampled + """ + input_text = '\r\n'.join([msg.content for msg in messages]) if messages else '' + output_text = response.message.content + + return await self.evaluate(input_text, output_text, sampling_ratio) + + def get_ai_config(self) -> AIJudgeConfig: + """ + Returns the AI Config used by this judge. + + :return: The judge AI configuration + """ + return self._ai_config + + def get_tracker(self) -> LDAIConfigTracker: + """ + Returns the tracker associated with this judge. + + :return: The tracker for the judge configuration + """ + return self._ai_config_tracker + + def get_provider(self) -> AIProvider: + """ + Returns the AI provider used by this judge. + + :return: The AI provider + """ + return self._ai_provider + + def _construct_evaluation_messages(self, input_text: str, output_text: str) -> List[LDMessage]: + """ + Constructs evaluation messages by combining judge's config messages with input/output. + + :param input_text: The input text + :param output_text: The output text to evaluate + :return: List of messages for evaluation + """ + if not self._ai_config.messages: + return [] + + messages: List[LDMessage] = [] + for msg in self._ai_config.messages: + # Interpolate message content with reserved variables + content = self._interpolate_message(msg.content, { + 'message_history': input_text, + 'response_to_evaluate': output_text, + }) + messages.append(LDMessage(role=msg.role, content=content)) + + return messages + + def _interpolate_message(self, content: str, variables: Dict[str, str]) -> str: + """ + Interpolates message content with variables using Mustache templating. + + :param content: The message content template + :param variables: Variables to interpolate + :return: Interpolated message content + """ + # Use chevron (Mustache) for templating, with no escaping + return chevron.render(content, variables) + + def _parse_evaluation_response(self, data: Dict[str, Any]) -> Dict[str, EvalScore]: + """ + Parses the structured evaluation response from the AI provider. + + :param data: The structured response data + :return: Dictionary of evaluation scores keyed by metric key + """ + results: Dict[str, EvalScore] = {} + + if not data.get('evaluations') or not isinstance(data['evaluations'], dict): + self._logger.warning('Invalid response: missing or invalid evaluations object') + return results + + evaluations = data['evaluations'] + + for metric_key in self._ai_config.evaluation_metric_keys: + evaluation = evaluations.get(metric_key) + + if not evaluation or not isinstance(evaluation, dict): + self._logger.warning(f'Missing evaluation for metric key: {metric_key}') + continue + + score = evaluation.get('score') + reasoning = evaluation.get('reasoning') + + if not isinstance(score, (int, float)) or score < 0 or score > 1: + self._logger.warning( + f'Invalid score evaluated for {metric_key}: {score}. ' + 'Score must be a number between 0 and 1 inclusive' + ) + continue + + if not isinstance(reasoning, str): + self._logger.warning( + f'Invalid reasoning evaluated for {metric_key}: {reasoning}. ' + 'Reasoning must be a string' + ) + continue + + results[metric_key] = EvalScore(score=float(score), reasoning=reasoning) + + return results diff --git a/packages/core/ldai/judge/evaluation_schema_builder.py b/packages/core/ldai/judge/evaluation_schema_builder.py new file mode 100644 index 0000000..8fbc712 --- /dev/null +++ b/packages/core/ldai/judge/evaluation_schema_builder.py @@ -0,0 +1,74 @@ +"""Internal class for building dynamic evaluation response schemas.""" + +from typing import Any, Dict, List + + +class EvaluationSchemaBuilder: + """ + Internal class for building dynamic evaluation response schemas. + Not exported - only used internally by Judge. + """ + + @staticmethod + def build(evaluation_metric_keys: list[str]) -> Dict[str, Any]: + """ + Build an evaluation response schema from evaluation metric keys. + + :param evaluation_metric_keys: List of evaluation metric keys + :return: Schema dictionary for structured output + """ + return { + 'title': 'EvaluationResponse', + 'description': f"Response containing evaluation results for {', '.join(evaluation_metric_keys)} metrics", + 'type': 'object', + 'properties': { + 'evaluations': { + 'type': 'object', + 'description': f"Object containing evaluation results for {', '.join(evaluation_metric_keys)} metrics", + 'properties': EvaluationSchemaBuilder._build_key_properties(evaluation_metric_keys), + 'required': evaluation_metric_keys, + 'additionalProperties': False, + }, + }, + 'required': ['evaluations'], + 'additionalProperties': False, + } + + @staticmethod + def _build_key_properties(evaluation_metric_keys: list[str]) -> Dict[str, Any]: + """ + Build properties for each evaluation metric key. + + :param evaluation_metric_keys: List of evaluation metric keys + :return: Dictionary of properties for each key + """ + result: Dict[str, Any] = {} + for key in evaluation_metric_keys: + result[key] = EvaluationSchemaBuilder._build_key_schema(key) + return result + + @staticmethod + def _build_key_schema(key: str) -> Dict[str, Any]: + """ + Build schema for a single evaluation metric key. + + :param key: Evaluation metric key + :return: Schema dictionary for the key + """ + return { + 'type': 'object', + 'properties': { + 'score': { + 'type': 'number', + 'minimum': 0, + 'maximum': 1, + 'description': f'Score between 0.0 and 1.0 for {key}', + }, + 'reasoning': { + 'type': 'string', + 'description': f'Reasoning behind the score for {key}', + }, + }, + 'required': ['score', 'reasoning'], + 'additionalProperties': False, + } diff --git a/packages/core/ldai/models.py b/packages/core/ldai/models.py new file mode 100644 index 0000000..c2abe56 --- /dev/null +++ b/packages/core/ldai/models.py @@ -0,0 +1,361 @@ +import warnings +from dataclasses import dataclass, field +from typing import Any, Dict, List, Literal, Optional, Union + +from ldai.tracker import LDAIConfigTracker + + +@dataclass +class LDMessage: + role: Literal['system', 'user', 'assistant'] + content: str + + def to_dict(self) -> dict: + """ + Render the given message as a dictionary object. + """ + return { + 'role': self.role, + 'content': self.content, + } + + +class ModelConfig: + """ + Configuration related to the model. + """ + + def __init__(self, name: str, parameters: Optional[Dict[str, Any]] = None, custom: Optional[Dict[str, Any]] = None): + """ + :param name: The name of the model. + :param parameters: Additional model-specific parameters. + :param custom: Additional customer provided data. + """ + self._name = name + self._parameters = parameters + self._custom = custom + + @property + def name(self) -> str: + """ + The name of the model. + """ + return self._name + + def get_parameter(self, key: str) -> Any: + """ + Retrieve model-specific parameters. + + Accessing a named, typed attribute (e.g. name) will result in the call + being delegated to the appropriate property. + """ + if key == 'name': + return self.name + + if self._parameters is None: + return None + + return self._parameters.get(key) + + def get_custom(self, key: str) -> Any: + """ + Retrieve customer provided data. + """ + if self._custom is None: + return None + + return self._custom.get(key) + + def to_dict(self) -> dict: + """ + Render the given model config as a dictionary object. + """ + return { + 'name': self._name, + 'parameters': self._parameters, + 'custom': self._custom, + } + + +class ProviderConfig: + """ + Configuration related to the provider. + """ + + def __init__(self, name: str): + self._name = name + + @property + def name(self) -> str: + """ + The name of the provider. + """ + return self._name + + def to_dict(self) -> dict: + """ + Render the given provider config as a dictionary object. + """ + return { + 'name': self._name, + } + + +# ============================================================================ +# Judge Types +# ============================================================================ + +@dataclass(frozen=True) +class JudgeConfiguration: + """ + Configuration for judge attachment to AI Configs. + """ + + @dataclass(frozen=True) + class Judge: + """ + Configuration for a single judge attachment. + """ + key: str + sampling_rate: float + + def to_dict(self) -> dict: + """ + Render the judge as a dictionary object. + """ + return { + 'key': self.key, + 'samplingRate': self.sampling_rate, + } + + judges: List['JudgeConfiguration.Judge'] + + def to_dict(self) -> dict: + """ + Render the judge configuration as a dictionary object. + """ + return { + 'judges': [judge.to_dict() for judge in self.judges], + } + + +# ============================================================================ +# Base AI Config Types +# ============================================================================ + +@dataclass(frozen=True) +class AIConfigDefault: + """ + Base AI Config interface for default implementations with optional enabled property. + """ + enabled: Optional[bool] = None + model: Optional[ModelConfig] = None + provider: Optional[ProviderConfig] = None + + def _base_to_dict(self) -> Dict[str, Any]: + """ + Render the base config fields as a dictionary object. + """ + return { + '_ldMeta': { + 'enabled': self.enabled or False, + }, + 'model': self.model.to_dict() if self.model else None, + 'provider': self.provider.to_dict() if self.provider else None, + } + + +@dataclass(frozen=True) +class AIConfig: + """ + Base AI Config interface without mode-specific fields. + """ + enabled: bool + model: Optional[ModelConfig] = None + provider: Optional[ProviderConfig] = None + tracker: Optional[LDAIConfigTracker] = None + + def _base_to_dict(self) -> Dict[str, Any]: + """ + Render the base config fields as a dictionary object. + """ + return { + '_ldMeta': { + 'enabled': self.enabled, + }, + 'model': self.model.to_dict() if self.model else None, + 'provider': self.provider.to_dict() if self.provider else None, + } + + +# ============================================================================ +# Completion Config Types +# ============================================================================ + +@dataclass(frozen=True) +class AICompletionConfigDefault(AIConfigDefault): + """ + Default Completion AI Config (default mode). + """ + messages: Optional[List[LDMessage]] = None + judge_configuration: Optional[JudgeConfiguration] = None + + def to_dict(self) -> dict: + """ + Render the given default values as an AICompletionConfigDefault-compatible dictionary object. + """ + result = self._base_to_dict() + result['messages'] = [message.to_dict() for message in self.messages] if self.messages else None + if self.judge_configuration is not None: + result['judgeConfiguration'] = self.judge_configuration.to_dict() + return result + + +@dataclass(frozen=True) +class AICompletionConfig(AIConfig): + """ + Completion AI Config (default mode). + """ + messages: Optional[List[LDMessage]] = None + judge_configuration: Optional[JudgeConfiguration] = None + + def to_dict(self) -> dict: + """ + Render the given completion config as a dictionary object. + """ + result = self._base_to_dict() + result['messages'] = [message.to_dict() for message in self.messages] if self.messages else None + if self.judge_configuration is not None: + result['judgeConfiguration'] = self.judge_configuration.to_dict() + return result + + +# ============================================================================ +# Agent Config Types +# ============================================================================ + +@dataclass(frozen=True) +class AIAgentConfigDefault(AIConfigDefault): + """ + Default Agent-specific AI Config with instructions. + """ + instructions: Optional[str] = None + judge_configuration: Optional[JudgeConfiguration] = None + + def to_dict(self) -> Dict[str, Any]: + """ + Render the given agent config default as a dictionary object. + """ + result = self._base_to_dict() + if self.instructions is not None: + result['instructions'] = self.instructions + if self.judge_configuration is not None: + result['judgeConfiguration'] = self.judge_configuration.to_dict() + return result + + +@dataclass(frozen=True) +class AIAgentConfig(AIConfig): + """ + Agent-specific AI Config with instructions. + """ + instructions: Optional[str] = None + judge_configuration: Optional[JudgeConfiguration] = None + + def to_dict(self) -> Dict[str, Any]: + """ + Render the given agent config as a dictionary object. + """ + result = self._base_to_dict() + if self.instructions is not None: + result['instructions'] = self.instructions + if self.judge_configuration is not None: + result['judgeConfiguration'] = self.judge_configuration.to_dict() + return result + + +# ============================================================================ +# Judge Config Types +# ============================================================================ + +@dataclass(frozen=True) +class AIJudgeConfigDefault(AIConfigDefault): + """ + Default Judge-specific AI Config with required evaluation metric key. + """ + messages: Optional[List[LDMessage]] = None + evaluation_metric_keys: Optional[List[str]] = None + + def to_dict(self) -> dict: + """ + Render the given judge config default as a dictionary object. + """ + result = self._base_to_dict() + result['messages'] = [message.to_dict() for message in self.messages] if self.messages else None + if self.evaluation_metric_keys is not None: + result['evaluationMetricKeys'] = self.evaluation_metric_keys + return result + + +@dataclass(frozen=True) +class AIJudgeConfig(AIConfig): + """ + Judge-specific AI Config with required evaluation metric key. + """ + evaluation_metric_keys: List[str] = field(default_factory=list) + messages: Optional[List[LDMessage]] = None + + def to_dict(self) -> dict: + """ + Render the given judge config as a dictionary object. + """ + result = self._base_to_dict() + result['evaluationMetricKeys'] = self.evaluation_metric_keys + result['messages'] = [message.to_dict() for message in self.messages] if self.messages else None + return result + + +# ============================================================================ +# Agent Request Config +# ============================================================================ + +@dataclass +class AIAgentConfigRequest: + """ + Configuration for a single agent request. + + Combines agent key with its specific default configuration and variables. + """ + key: str + default_value: AIAgentConfigDefault + variables: Optional[Dict[str, Any]] = None + + +# Type alias for multiple agents +AIAgents = Dict[str, AIAgentConfig] + +# Type alias for all AI Config variants +AIConfigKind = Union[AIAgentConfig, AICompletionConfig, AIJudgeConfig] + + +# ============================================================================ +# Deprecated Type Aliases for Backward Compatibility +# ============================================================================ + +# Note: These are type aliases that point to the new types. +# Since Python uses duck typing, these will work at runtime even if type checkers complain. +# The old AIConfig had optional enabled, so it maps to AICompletionConfigDefault +# The old AIConfig return type had required enabled, so it maps to AICompletionConfig + +# Note: AIConfig is now the base class for all config types (defined above at line 169) +# For default configs (with optional enabled), use AICompletionConfigDefault instead +# For required configs (with required enabled), use AICompletionConfig instead + +# Deprecated: Use AIAgentConfigDefault instead +LDAIAgentDefaults = AIAgentConfigDefault + +# Deprecated: Use AIAgentConfigRequest instead +LDAIAgentConfig = AIAgentConfigRequest + +# Deprecated: Use AIAgentConfig instead (note: this was the old return type) +LDAIAgent = AIAgentConfig diff --git a/packages/core/ldai/providers/ai_provider.py b/packages/core/ldai/providers/ai_provider.py new file mode 100644 index 0000000..637bb9d --- /dev/null +++ b/packages/core/ldai/providers/ai_provider.py @@ -0,0 +1,88 @@ +"""Abstract base class for AI providers.""" + +import logging +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional + +from ldai.models import AIConfigKind, LDMessage +from ldai.providers.types import ChatResponse, LDAIMetrics, StructuredResponse + + +class AIProvider(ABC): + """ + Abstract base class for AI providers that implement chat model functionality. + + This class provides the contract that all provider implementations must follow + to integrate with LaunchDarkly's tracking and configuration capabilities. + + Following the AICHAT spec recommendation to use base classes with non-abstract methods + for better extensibility and backwards compatibility. + """ + + def __init__(self): + """ + Initialize the AI provider. + + Creates a logger for this provider instance. + """ + self._logger = logging.getLogger(f'{__name__}.{self.__class__.__name__}') + + async def invoke_model(self, messages: List[LDMessage]) -> ChatResponse: + """ + Invoke the chat model with an array of messages. + + This method should convert messages to provider format, invoke the model, + and return a ChatResponse with the result and metrics. + + Default implementation takes no action and returns a placeholder response. + Provider implementations should override this method. + + :param messages: Array of LDMessage objects representing the conversation + :return: ChatResponse containing the model's response + """ + self._logger.warning('invokeModel not implemented by this provider') + + return ChatResponse( + message=LDMessage(role='assistant', content=''), + metrics=LDAIMetrics(success=False, usage=None), + ) + + async def invoke_structured_model( + self, + messages: List[LDMessage], + response_structure: Dict[str, Any], + ) -> StructuredResponse: + """ + Invoke the chat model with structured output support. + + This method should convert messages to provider format, invoke the model with + structured output configuration, and return a structured response. + + Default implementation takes no action and returns a placeholder response. + Provider implementations should override this method. + + :param messages: Array of LDMessage objects representing the conversation + :param response_structure: Dictionary of output configurations keyed by output name + :return: StructuredResponse containing the structured data + """ + self._logger.warning('invokeStructuredModel not implemented by this provider') + + return StructuredResponse( + data={}, + raw_response='', + metrics=LDAIMetrics(success=False, usage=None), + ) + + @staticmethod + @abstractmethod + async def create(ai_config: AIConfigKind) -> 'AIProvider': + """ + Static method that constructs an instance of the provider. + + Each provider implementation must provide their own static create method + that accepts an AIConfigKind and returns a configured instance. + + :param ai_config: The LaunchDarkly AI configuration + :return: Configured provider instance + """ + raise NotImplementedError('Provider implementations must override the static create method') diff --git a/packages/core/ldai/providers/ai_provider_factory.py b/packages/core/ldai/providers/ai_provider_factory.py new file mode 100644 index 0000000..7c1dec2 --- /dev/null +++ b/packages/core/ldai/providers/ai_provider_factory.py @@ -0,0 +1,160 @@ +"""Factory for creating AIProvider instances based on the provider configuration.""" + +import importlib +import logging +from typing import List, Literal, Optional, Type + +from ldai.models import AIConfigKind +from ldai.providers.ai_provider import AIProvider + +# List of supported AI providers +SUPPORTED_AI_PROVIDERS = [ + # Multi-provider packages should be last in the list + 'langchain', +] + +# Type representing the supported AI providers +SupportedAIProvider = Literal['langchain'] + + +class AIProviderFactory: + """ + Factory for creating AIProvider instances based on the provider configuration. + """ + + _logger = logging.getLogger(__name__) + + @staticmethod + async def create( + ai_config: AIConfigKind, + default_ai_provider: Optional[SupportedAIProvider] = None, + ) -> Optional[AIProvider]: + """ + Create an AIProvider instance based on the AI configuration. + + This method attempts to load provider-specific implementations dynamically. + Returns None if the provider is not supported. + + :param ai_config: The AI configuration + :param default_ai_provider: Optional default AI provider to use + :return: AIProvider instance or None if not supported + """ + provider_name = ai_config.provider.name.lower() if ai_config.provider else None + # Determine which providers to try based on default_ai_provider + providers_to_try = AIProviderFactory._get_providers_to_try(default_ai_provider, provider_name) + + # Try each provider in order + for provider_type in providers_to_try: + provider = await AIProviderFactory._try_create_provider(provider_type, ai_config) + if provider: + return provider + + # If no provider was successfully created, log a warning + AIProviderFactory._logger.warning( + f"Provider is not supported or failed to initialize: {provider_name or 'unknown'}" + ) + return None + + @staticmethod + def _get_providers_to_try( + default_ai_provider: Optional[SupportedAIProvider], + provider_name: Optional[str], + ) -> List[SupportedAIProvider]: + """ + Determine which providers to try based on default_ai_provider and provider_name. + + :param default_ai_provider: Optional default provider to use + :param provider_name: Optional provider name from config + :return: List of providers to try in order + """ + # If default_ai_provider is set, only try that specific provider + if default_ai_provider: + return [default_ai_provider] + + # If no default_ai_provider is set, try all providers in order + provider_set = set() + + # First try the specific provider if it's supported + if provider_name and provider_name in SUPPORTED_AI_PROVIDERS: + provider_set.add(provider_name) # type: ignore + + # Then try multi-provider packages, but avoid duplicates + multi_provider_packages: List[SupportedAIProvider] = ['langchain'] + for provider in multi_provider_packages: + provider_set.add(provider) + + return list(provider_set) # type: ignore[arg-type] + + @staticmethod + async def _try_create_provider( + provider_type: SupportedAIProvider, + ai_config: AIConfigKind, + ) -> Optional[AIProvider]: + """ + Try to create a provider of the specified type. + + :param provider_type: Type of provider to create + :param ai_config: AI configuration + :return: AIProvider instance or None if creation failed + """ + # Handle built-in providers (part of this package) + if provider_type == 'langchain': + try: + from ldai.providers.langchain import LangChainProvider + return await LangChainProvider.create(ai_config) + except ImportError as error: + AIProviderFactory._logger.warning( + f"Error creating LangChainProvider: {error}. " + f"Make sure langchain and langchain-core packages are installed." + ) + return None + + # TODO: REL-10773 OpenAI provider + # TODO: REL-10776 Vercel provider + # For future external providers, use dynamic import + provider_mappings = { + # 'openai': ('launchdarkly_server_sdk_ai_openai', 'OpenAIProvider'), + # 'vercel': ('launchdarkly_server_sdk_ai_vercel', 'VercelProvider'), + } + + if provider_type not in provider_mappings: + return None + + package_name, provider_class_name = provider_mappings[provider_type] + return await AIProviderFactory._create_provider( + package_name, provider_class_name, ai_config + ) + + @staticmethod + async def _create_provider( + package_name: str, + provider_class_name: str, + ai_config: AIConfigKind, + ) -> Optional[AIProvider]: + """ + Create a provider instance dynamically. + + :param package_name: Name of the package containing the provider + :param provider_class_name: Name of the provider class + :param ai_config: AI configuration + :return: AIProvider instance or None if creation failed + """ + try: + # Try to dynamically import the provider + # This will work if the package is installed + module = importlib.import_module(package_name) + provider_class: Type[AIProvider] = getattr(module, provider_class_name) + + provider = await provider_class.create(ai_config) + AIProviderFactory._logger.debug( + f"Successfully created AIProvider for: {ai_config.provider.name if ai_config.provider else 'unknown'} " + f"with package {package_name}" + ) + return provider + except (ImportError, AttributeError, Exception) as error: + # If the provider is not available or creation fails, return None + AIProviderFactory._logger.warning( + f"Error creating AIProvider for: {ai_config.provider.name if ai_config.provider else 'unknown'} " + f"with package {package_name}: {error}" + ) + return None diff --git a/packages/core/ldai/providers/types.py b/packages/core/ldai/providers/types.py new file mode 100644 index 0000000..de54698 --- /dev/null +++ b/packages/core/ldai/providers/types.py @@ -0,0 +1,91 @@ +"""Types for AI provider responses.""" + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from ldai.models import LDMessage +from ldai.tracker import TokenUsage + + +@dataclass +class LDAIMetrics: + """ + Metrics information for AI operations that includes success status and token usage. + """ + success: bool + usage: Optional[TokenUsage] = None + + def to_dict(self) -> Dict[str, Any]: + """ + Render the metrics as a dictionary object. + """ + result: Dict[str, Any] = { + 'success': self.success, + } + if self.usage is not None: + result['usage'] = { + 'total': self.usage.total, + 'input': self.usage.input, + 'output': self.usage.output, + } + return result + + +@dataclass +class ChatResponse: + """ + Chat response structure. + """ + message: LDMessage + metrics: LDAIMetrics + evaluations: Optional[List[Any]] = None # List of JudgeResponse, will be populated later + + +@dataclass +class StructuredResponse: + """ + Structured response from AI models. + """ + data: Dict[str, Any] + raw_response: str + metrics: LDAIMetrics + + +@dataclass +class EvalScore: + """ + Score and reasoning for a single evaluation metric. + """ + score: float # Score between 0.0 and 1.0 + reasoning: str # Reasoning behind the provided score + + def to_dict(self) -> Dict[str, Any]: + """ + Render the evaluation score as a dictionary object. + """ + return { + 'score': self.score, + 'reasoning': self.reasoning, + } + + +@dataclass +class JudgeResponse: + """ + Response from a judge evaluation containing scores and reasoning for multiple metrics. + """ + evals: Dict[str, EvalScore] # Dictionary where keys are metric names and values contain score and reasoning + success: bool # Whether the evaluation completed successfully + error: Optional[str] = None # Error message if evaluation failed + + def to_dict(self) -> Dict[str, Any]: + """ + Render the judge response as a dictionary object. + """ + result: Dict[str, Any] = { + 'evals': {key: eval_score.to_dict() for key, eval_score in self.evals.items()}, + 'success': self.success, + } + if self.error is not None: + result['error'] = self.error + return result diff --git a/ldai/tracker.py b/packages/core/ldai/tracker.py similarity index 71% rename from ldai/tracker.py rename to packages/core/ldai/tracker.py index a049952..11b846a 100644 --- a/ldai/tracker.py +++ b/packages/core/ldai/tracker.py @@ -1,7 +1,7 @@ import time from dataclasses import dataclass from enum import Enum -from typing import Dict, Optional +from typing import Any, Dict, Optional from ldclient import Context, LDClient @@ -144,7 +144,7 @@ def track_duration_of(self, func): An exception occurring during the execution of the function will still track the duration. The exception will be re-thrown. - :param func: Function to track. + :param func: Function to track (synchronous only). :return: Result of the tracked function. """ start_time = time.time() @@ -157,6 +157,90 @@ def track_duration_of(self, func): return result + async def track_metrics_of(self, metrics_extractor, func): + """ + Track metrics for a generic AI operation. + + This function will track the duration of the operation, extract metrics using the provided + metrics extractor function, and track success or error status accordingly. + + If the provided function throws, then this method will also throw. + In the case the provided function throws, this function will record the duration and an error. + A failed operation will not have any token usage data. + + :param metrics_extractor: Function that extracts LDAIMetrics from the operation result + :param func: Async function which executes the operation + :return: The result of the operation + """ + start_time = time.time() + result = None + try: + result = await func() + except Exception as err: + end_time = time.time() + duration = int((end_time - start_time) * 1000) + self.track_duration(duration) + self.track_error() + raise err + + # Track duration after successful call + end_time = time.time() + duration = int((end_time - start_time) * 1000) + self.track_duration(duration) + + # Extract metrics after successful AI call + from ldai.providers.types import LDAIMetrics + metrics = metrics_extractor(result) + + # Track success/error based on metrics + if metrics.success: + self.track_success() + else: + self.track_error() + + # Track token usage if available + if metrics.usage: + self.track_tokens(metrics.usage) + + return result + + def track_eval_scores(self, scores: Dict[str, Any]) -> None: + """ + Track evaluation scores for multiple metrics. + + :param scores: Dictionary mapping metric keys to their evaluation scores (EvalScore objects) + """ + from ldai.providers.types import EvalScore + + # Track each evaluation score individually + for metric_key, eval_score in scores.items(): + if isinstance(eval_score, EvalScore): + self._ld_client.track( + metric_key, + self._context, + self.__get_track_data(), + eval_score.score + ) + + def track_judge_response(self, judge_response: Any) -> None: + """ + Track a judge response, including evaluation scores and success status. + + :param judge_response: JudgeResponse object containing evals and success status + """ + from ldai.providers.types import JudgeResponse + + if isinstance(judge_response, JudgeResponse): + # Track evaluation scores + if judge_response.evals: + self.track_eval_scores(judge_response.evals) + + # Track success/error based on judge response + if judge_response.success: + self.track_success() + else: + self.track_error() + def track_feedback(self, feedback: Dict[str, FeedbackKind]) -> None: """ Track user feedback for an AI operation. @@ -197,7 +281,7 @@ def track_error(self) -> None: "$ld:ai:generation:error", self._context, self.__get_track_data(), 1 ) - def track_openai_metrics(self, func): + async def track_openai_metrics(self, func): """ Track OpenAI-specific operations. @@ -211,15 +295,22 @@ def track_openai_metrics(self, func): A failed operation will not have any token usage data. - :param func: Function to track. + :param func: Async function to track. :return: Result of the tracked function. """ + start_time = time.time() try: - result = self.track_duration_of(func) + result = await func() + end_time = time.time() + duration = int((end_time - start_time) * 1000) + self.track_duration(duration) self.track_success() if hasattr(result, "usage") and hasattr(result.usage, "to_dict"): self.track_tokens(_openai_to_token_usage(result.usage.to_dict())) except Exception: + end_time = time.time() + duration = int((end_time - start_time) * 1000) + self.track_duration(duration) self.track_error() raise diff --git a/pyproject.toml b/packages/core/pyproject.toml similarity index 96% rename from pyproject.toml rename to packages/core/pyproject.toml index 200215c..4cc0756 100644 --- a/pyproject.toml +++ b/packages/core/pyproject.toml @@ -22,9 +22,6 @@ classifiers = [ "Topic :: Software Development :: Libraries", ] packages = [ { include = "ldai" } ] -exclude = [ - { path = "ldai/testing", format = "wheel" } -] [tool.poetry.dependencies] python = ">=3.9,<4" @@ -36,6 +33,7 @@ chevron = "=0.14.0" pytest = ">=2.8" pytest-cov = ">=2.4.0" pytest-mypy = "==1.0.1" +pytest-asyncio = ">=0.21.0" mypy = "==1.18.2" pycodestyle = "^2.12.1" isort = ">=5.13.2,<7.0.0" diff --git a/packages/core/tests/__init__.py b/packages/core/tests/__init__.py new file mode 100644 index 0000000..1f7baa7 --- /dev/null +++ b/packages/core/tests/__init__.py @@ -0,0 +1,2 @@ +"""Tests for LaunchDarkly Server SDK for AI - Core package.""" + diff --git a/ldai/testing/test_agents.py b/packages/core/tests/test_agents.py similarity index 98% rename from ldai/testing/test_agents.py rename to packages/core/tests/test_agents.py index b2e80c0..755f2e5 100644 --- a/ldai/testing/test_agents.py +++ b/packages/core/tests/test_agents.py @@ -2,8 +2,8 @@ from ldclient import Config, Context, LDClient from ldclient.integrations.test_data import TestData -from ldai.client import (LDAIAgentConfig, LDAIAgentDefaults, LDAIClient, - ModelConfig, ProviderConfig) +from ldai import (LDAIAgentConfig, LDAIAgentDefaults, LDAIClient, ModelConfig, + ProviderConfig) @pytest.fixture diff --git a/ldai/testing/test_model_config.py b/packages/core/tests/test_model_config.py similarity index 84% rename from ldai/testing/test_model_config.py rename to packages/core/tests/test_model_config.py index 1ffc033..d556c10 100644 --- a/ldai/testing/test_model_config.py +++ b/packages/core/tests/test_model_config.py @@ -2,7 +2,7 @@ from ldclient import Config, Context, LDClient from ldclient.integrations.test_data import TestData -from ldai.client import AIConfig, LDAIClient, LDMessage, ModelConfig +from ldai import AICompletionConfigDefault, LDAIClient, LDMessage, ModelConfig @pytest.fixture @@ -133,14 +133,14 @@ def test_model_config_handles_custom(): def test_uses_default_on_invalid_flag(ldai_client: LDAIClient): context = Context.create('user-key') - default_value = AIConfig( + default_value = AICompletionConfigDefault( enabled=True, model=ModelConfig('fakeModel', parameters={'temperature': 0.5, 'maxTokens': 4096}), messages=[LDMessage(role='system', content='Hello, {{name}}!')], ) variables = {'name': 'World'} - config, _ = ldai_client.config('missing-flag', context, default_value, variables) + config = ldai_client.config('missing-flag', context, default_value, variables) assert config.messages is not None assert len(config.messages) > 0 @@ -155,14 +155,14 @@ def test_uses_default_on_invalid_flag(ldai_client: LDAIClient): def test_model_config_interpolation(ldai_client: LDAIClient): context = Context.create('user-key') - default_value = AIConfig( + default_value = AICompletionConfigDefault( enabled=True, model=ModelConfig('fakeModel'), messages=[LDMessage(role='system', content='Hello, {{name}}!')], ) variables = {'name': 'World'} - config, _ = ldai_client.config('model-config', context, default_value, variables) + config = ldai_client.config('model-config', context, default_value, variables) assert config.messages is not None assert len(config.messages) > 0 @@ -177,9 +177,9 @@ def test_model_config_interpolation(ldai_client: LDAIClient): def test_model_config_no_variables(ldai_client: LDAIClient): context = Context.create('user-key') - default_value = AIConfig(enabled=True, model=ModelConfig('fake-model'), messages=[]) + default_value = AICompletionConfigDefault(enabled=True, model=ModelConfig('fake-model'), messages=[]) - config, _ = ldai_client.config('model-config', context, default_value, {}) + config = ldai_client.config('model-config', context, default_value, {}) assert config.messages is not None assert len(config.messages) > 0 @@ -194,10 +194,10 @@ def test_model_config_no_variables(ldai_client: LDAIClient): def test_provider_config_handling(ldai_client: LDAIClient): context = Context.builder('user-key').name("Sandy").build() - default_value = AIConfig(enabled=True, model=ModelConfig('fake-model'), messages=[]) + default_value = AICompletionConfigDefault(enabled=True, model=ModelConfig('fake-model'), messages=[]) variables = {'name': 'World'} - config, _ = ldai_client.config('model-config', context, default_value, variables) + config = ldai_client.config('model-config', context, default_value, variables) assert config.provider is not None assert config.provider.name == 'fakeProvider' @@ -205,10 +205,10 @@ def test_provider_config_handling(ldai_client: LDAIClient): def test_context_interpolation(ldai_client: LDAIClient): context = Context.builder('user-key').name("Sandy").set('last', 'Beaches').build() - default_value = AIConfig(enabled=True, model=ModelConfig('fake-model'), messages=[]) + default_value = AICompletionConfigDefault(enabled=True, model=ModelConfig('fake-model'), messages=[]) variables = {'name': 'World'} - config, _ = ldai_client.config( + config = ldai_client.config( 'ctx-interpolation', context, default_value, variables ) @@ -228,10 +228,10 @@ def test_multi_context_interpolation(ldai_client: LDAIClient): user_context = Context.builder('user-key').name("Sandy").build() org_context = Context.builder('org-key').kind('org').name("LaunchDarkly").set('shortname', 'LD').build() context = Context.multi_builder().add(user_context).add(org_context).build() - default_value = AIConfig(enabled=True, model=ModelConfig('fake-model'), messages=[]) + default_value = AICompletionConfigDefault(enabled=True, model=ModelConfig('fake-model'), messages=[]) variables = {'name': 'World'} - config, _ = ldai_client.config( + config = ldai_client.config( 'multi-ctx-interpolation', context, default_value, variables ) @@ -249,10 +249,10 @@ def test_multi_context_interpolation(ldai_client: LDAIClient): def test_model_config_multiple(ldai_client: LDAIClient): context = Context.create('user-key') - default_value = AIConfig(enabled=True, model=ModelConfig('fake-model'), messages=[]) + default_value = AICompletionConfigDefault(enabled=True, model=ModelConfig('fake-model'), messages=[]) variables = {'name': 'World', 'day': 'Monday'} - config, _ = ldai_client.config( + config = ldai_client.config( 'multiple-messages', context, default_value, variables ) @@ -270,9 +270,9 @@ def test_model_config_multiple(ldai_client: LDAIClient): def test_model_config_disabled(ldai_client: LDAIClient): context = Context.create('user-key') - default_value = AIConfig(enabled=False, model=ModelConfig('fake-model'), messages=[]) + default_value = AICompletionConfigDefault(enabled=False, model=ModelConfig('fake-model'), messages=[]) - config, _ = ldai_client.config('off-config', context, default_value, {}) + config = ldai_client.config('off-config', context, default_value, {}) assert config.model is not None assert config.enabled is False @@ -283,9 +283,9 @@ def test_model_config_disabled(ldai_client: LDAIClient): def test_model_initial_config_disabled(ldai_client: LDAIClient): context = Context.create('user-key') - default_value = AIConfig(enabled=False, model=ModelConfig('fake-model'), messages=[]) + default_value = AICompletionConfigDefault(enabled=False, model=ModelConfig('fake-model'), messages=[]) - config, _ = ldai_client.config('initial-config-disabled', context, default_value, {}) + config = ldai_client.config('initial-config-disabled', context, default_value, {}) assert config.enabled is False assert config.model is None @@ -295,9 +295,9 @@ def test_model_initial_config_disabled(ldai_client: LDAIClient): def test_model_initial_config_enabled(ldai_client: LDAIClient): context = Context.create('user-key') - default_value = AIConfig(enabled=False, model=ModelConfig('fake-model'), messages=[]) + default_value = AICompletionConfigDefault(enabled=False, model=ModelConfig('fake-model'), messages=[]) - config, _ = ldai_client.config('initial-config-enabled', context, default_value, {}) + config = ldai_client.config('initial-config-enabled', context, default_value, {}) assert config.enabled is True assert config.model is None @@ -318,9 +318,9 @@ def test_config_method_tracking(ldai_client: LDAIClient): client = LDAIClient(mock_client) context = Context.create('user-key') - default_value = AIConfig(enabled=False, model=ModelConfig('fake-model'), messages=[]) + default_value = AICompletionConfigDefault(enabled=False, model=ModelConfig('fake-model'), messages=[]) - config, tracker = client.config('test-config-key', context, default_value) + config = client.config('test-config-key', context, default_value) mock_client.track.assert_called_once_with( '$ld:ai:config:function:single', diff --git a/ldai/testing/test_tracker.py b/packages/core/tests/test_tracker.py similarity index 97% rename from ldai/testing/test_tracker.py rename to packages/core/tests/test_tracker.py index 19c8161..2e39d98 100644 --- a/ldai/testing/test_tracker.py +++ b/packages/core/tests/test_tracker.py @@ -276,7 +276,8 @@ def test_tracks_bedrock_metrics_with_error(client: LDClient): assert tracker.get_summary().usage == TokenUsage(330, 220, 110) -def test_tracks_openai_metrics(client: LDClient): +@pytest.mark.asyncio +async def test_tracks_openai_metrics(client: LDClient): context = Context.create("user-key") tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, "fakeModel", "fakeProvider", context) @@ -292,7 +293,10 @@ def to_dict(self): "completion_tokens": 110, } - tracker.track_openai_metrics(lambda: Result()) + async def get_result(): + return Result() + + await tracker.track_openai_metrics(get_result) calls = [ call( @@ -326,15 +330,16 @@ def to_dict(self): assert tracker.get_summary().usage == TokenUsage(330, 220, 110) -def test_tracks_openai_metrics_with_exception(client: LDClient): +@pytest.mark.asyncio +async def test_tracks_openai_metrics_with_exception(client: LDClient): context = Context.create("user-key") tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, "fakeModel", "fakeProvider", context) - def raise_exception(): + async def raise_exception(): raise ValueError("Something went wrong") try: - tracker.track_openai_metrics(raise_exception) + await tracker.track_openai_metrics(raise_exception) assert False, "Should have thrown an exception" except ValueError: pass diff --git a/packages/langchain/README.md b/packages/langchain/README.md new file mode 100644 index 0000000..06fd329 --- /dev/null +++ b/packages/langchain/README.md @@ -0,0 +1,58 @@ +# LaunchDarkly AI SDK - LangChain Provider + +This package provides LangChain provider support for the LaunchDarkly AI SDK. + +## Installation + +```bash +pip install launchdarkly-server-sdk-ai-langchain +``` + +This will automatically install the core SDK (`launchdarkly-server-sdk-ai`) and LangChain dependencies. + +## Usage + +```python +from ldclient import init, Context +from ldai import init_ai + +# Initialize clients +ld_client = init('your-sdk-key') +ai_client = init_ai(ld_client) + +# Create a chat - will automatically use LangChain provider +context = Context.create('user-key') +chat = await ai_client.create_chat('chat-config', context, { + 'enabled': True, + 'provider': {'name': 'openai'}, + 'model': {'name': 'gpt-4'} +}) + +if chat: + response = await chat.invoke('Hello!') + print(response.message.content) +``` + +## Supported LangChain Providers + +This provider supports any LangChain-compatible model, including: +- OpenAI (GPT-3.5, GPT-4, etc.) +- Anthropic (Claude) +- Google (Gemini) +- And many more through LangChain integrations + +## Requirements + +- Python 3.9+ +- launchdarkly-server-sdk-ai >= 0.10.1 +- langchain >= 0.3.0 +- langchain-core >= 0.3.0 + +## Documentation + +For full documentation, visit: https://docs.launchdarkly.com/sdk/ai/python + +## License + +Apache-2.0 + diff --git a/packages/langchain/ldai/providers/langchain/__init__.py b/packages/langchain/ldai/providers/langchain/__init__.py new file mode 100644 index 0000000..822f049 --- /dev/null +++ b/packages/langchain/ldai/providers/langchain/__init__.py @@ -0,0 +1,5 @@ +"""LangChain provider module for LaunchDarkly AI SDK.""" + +from ldai.providers.langchain.langchain_provider import LangChainProvider + +__all__ = ['LangChainProvider'] diff --git a/packages/langchain/ldai/providers/langchain/langchain_provider.py b/packages/langchain/ldai/providers/langchain/langchain_provider.py new file mode 100644 index 0000000..22542fd --- /dev/null +++ b/packages/langchain/ldai/providers/langchain/langchain_provider.py @@ -0,0 +1,264 @@ +"""LangChain implementation of AIProvider for LaunchDarkly AI SDK.""" + +import json +import logging +from typing import Any, Dict, List, Optional + +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage + +from ldai.models import AIConfigKind, LDMessage +from ldai.providers.ai_provider import AIProvider +from ldai.providers.types import ChatResponse, LDAIMetrics, StructuredResponse +from ldai.tracker import TokenUsage + + +class LangChainProvider(AIProvider): + """ + LangChain implementation of AIProvider. + + This provider integrates LangChain models with LaunchDarkly's tracking capabilities. + """ + + def __init__(self, llm: BaseChatModel): + """ + Initialize the LangChain provider. + + :param llm: LangChain BaseChatModel instance + """ + super().__init__() + self._llm = llm + + @staticmethod + async def create(ai_config: AIConfigKind) -> 'LangChainProvider': + """ + Static factory method to create a LangChain AIProvider from an AI configuration. + + :param ai_config: The LaunchDarkly AI configuration + :return: Configured LangChainProvider instance + """ + llm = await LangChainProvider.create_langchain_model(ai_config) + return LangChainProvider(llm) + + async def invoke_model(self, messages: List[LDMessage]) -> ChatResponse: + """ + Invoke the LangChain model with an array of messages. + + :param messages: Array of LDMessage objects representing the conversation + :return: ChatResponse containing the model's response + """ + try: + # Convert LDMessage[] to LangChain messages + langchain_messages = LangChainProvider.convert_messages_to_langchain(messages) + + # Get the LangChain response + response: AIMessage = await self._llm.ainvoke(langchain_messages) + + # Generate metrics early (assumes success by default) + metrics = LangChainProvider.get_ai_metrics_from_response(response) + + # Extract text content from the response + content: str = '' + if isinstance(response.content, str): + content = response.content + else: + # Log warning for non-string content (likely multimodal) + self._logger.warning( + f"Multimodal response not supported, expecting a string. " + f"Content type: {type(response.content)}, Content: {response.content}" + ) + # Update metrics to reflect content loss + metrics.success = False + + # Create the assistant message + assistant_message = LDMessage(role='assistant', content=content) + + return ChatResponse( + message=assistant_message, + metrics=metrics, + ) + except Exception as error: + self._logger.warning(f'LangChain model invocation failed: {error}') + + return ChatResponse( + message=LDMessage(role='assistant', content=''), + metrics=LDAIMetrics(success=False, usage=None), + ) + + async def invoke_structured_model( + self, + messages: List[LDMessage], + response_structure: Dict[str, Any], + ) -> StructuredResponse: + """ + Invoke the LangChain model with structured output support. + + :param messages: Array of LDMessage objects representing the conversation + :param response_structure: Dictionary of output configurations keyed by output name + :return: StructuredResponse containing the structured data + """ + try: + # Convert LDMessage[] to LangChain messages + langchain_messages = LangChainProvider.convert_messages_to_langchain(messages) + + # Get the LangChain response with structured output + # Note: with_structured_output is available on BaseChatModel in newer LangChain versions + if hasattr(self._llm, 'with_structured_output'): + structured_llm = self._llm.with_structured_output(response_structure) + response = await structured_llm.ainvoke(langchain_messages) + else: + # Fallback: invoke normally and try to parse as JSON + response_obj = await self._llm.ainvoke(langchain_messages) + if isinstance(response_obj, AIMessage): + try: + response = json.loads(response_obj.content) + except json.JSONDecodeError: + response = {'content': response_obj.content} + else: + response = response_obj + + # Using structured output doesn't support metrics + metrics = LDAIMetrics( + success=True, + usage=TokenUsage(total=0, input=0, output=0), + ) + + return StructuredResponse( + data=response if isinstance(response, dict) else {'result': response}, + raw_response=json.dumps(response) if not isinstance(response, str) else response, + metrics=metrics, + ) + except Exception as error: + self._logger.warning(f'LangChain structured model invocation failed: {error}') + + return StructuredResponse( + data={}, + raw_response='', + metrics=LDAIMetrics( + success=False, + usage=TokenUsage(total=0, input=0, output=0), + ), + ) + + def get_chat_model(self) -> BaseChatModel: + """ + Get the underlying LangChain model instance. + + :return: The LangChain BaseChatModel instance + """ + return self._llm + + @staticmethod + def map_provider(ld_provider_name: str) -> str: + """ + Map LaunchDarkly provider names to LangChain provider names. + + This method enables seamless integration between LaunchDarkly's standardized + provider naming and LangChain's naming conventions. + + :param ld_provider_name: LaunchDarkly provider name + :return: LangChain provider name + """ + lowercased_name = ld_provider_name.lower() + + mapping: Dict[str, str] = { + 'gemini': 'google-genai', + } + + return mapping.get(lowercased_name, lowercased_name) + + @staticmethod + def get_ai_metrics_from_response(response: AIMessage) -> LDAIMetrics: + """ + Get AI metrics from a LangChain provider response. + + This method extracts token usage information and success status from LangChain responses + and returns a LaunchDarkly LDAIMetrics object. + + :param response: The response from the LangChain model + :return: LDAIMetrics with success status and token usage + """ + # Extract token usage if available + usage: Optional[TokenUsage] = None + if hasattr(response, 'response_metadata') and response.response_metadata: + token_usage = response.response_metadata.get('token_usage') + if token_usage: + usage = TokenUsage( + total=token_usage.get('total_tokens') or token_usage.get('totalTokens') or 0, + input=token_usage.get('prompt_tokens') or token_usage.get('promptTokens') or 0, + output=token_usage.get('completion_tokens') or token_usage.get('completionTokens') or 0, + ) + + # LangChain responses that complete successfully are considered successful by default + return LDAIMetrics(success=True, usage=usage) + + @staticmethod + def convert_messages_to_langchain(messages: List[LDMessage]) -> List[BaseMessage]: + """ + Convert LaunchDarkly messages to LangChain messages. + + This helper method enables developers to work directly with LangChain message types + while maintaining compatibility with LaunchDarkly's standardized message format. + + :param messages: List of LDMessage objects + :return: List of LangChain message objects + """ + result: List[BaseMessage] = [] + for msg in messages: + if msg.role == 'system': + result.append(SystemMessage(content=msg.content)) + elif msg.role == 'user': + result.append(HumanMessage(content=msg.content)) + elif msg.role == 'assistant': + result.append(AIMessage(content=msg.content)) + else: + raise ValueError(f'Unsupported message role: {msg.role}') + return result + + @staticmethod + async def create_langchain_model(ai_config: AIConfigKind) -> BaseChatModel: + """ + Create a LangChain model from an AI configuration. + + This public helper method enables developers to initialize their own LangChain models + using LaunchDarkly AI configurations. + + :param ai_config: The LaunchDarkly AI configuration + :return: A configured LangChain BaseChatModel + """ + model_name = ai_config.model.name if ai_config.model else '' + provider = ai_config.provider.name if ai_config.provider else '' + parameters = ai_config.model.get_parameter('parameters') if ai_config.model else {} + if not isinstance(parameters, dict): + parameters = {} + + # Use LangChain's init_chat_model to support multiple providers + # Note: This requires langchain package to be installed + try: + # Try to import init_chat_model from langchain.chat_models + # This is available in langchain >= 0.1.0 + try: + from langchain.chat_models import init_chat_model + except ImportError: + # Fallback for older versions or different import path + from langchain.chat_models.universal import init_chat_model + + # Map provider name + langchain_provider = LangChainProvider.map_provider(provider) + + # Create model configuration + model_kwargs = {**parameters} + if langchain_provider: + model_kwargs['model_provider'] = langchain_provider + + # Initialize the chat model (init_chat_model may be async or sync) + result = init_chat_model(model_name, **model_kwargs) + # Handle both sync and async initialization + if hasattr(result, '__await__'): + return await result + return result + except ImportError as e: + raise ImportError( + 'langchain package is required for LangChainProvider. ' + 'Install it with: pip install langchain langchain-core' + ) from e diff --git a/packages/langchain/pyproject.toml b/packages/langchain/pyproject.toml new file mode 100644 index 0000000..33d2b3c --- /dev/null +++ b/packages/langchain/pyproject.toml @@ -0,0 +1,40 @@ +[tool.poetry] +name = "launchdarkly-server-sdk-ai-langchain" +version = "0.1.0" +description = "LangChain provider for LaunchDarkly AI SDK" +authors = ["LaunchDarkly "] +license = "Apache-2.0" +readme = "README.md" +homepage = "https://docs.launchdarkly.com/sdk/ai/python" +repository = "https://github.com/launchdarkly/python-server-sdk-ai" +documentation = "https://launchdarkly-python-sdk-ai.readthedocs.io/en/latest/" +classifiers = [ + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", +] +packages = [ { include = "ldai" } ] + +[tool.poetry.dependencies] +python = ">=3.9,<4" +launchdarkly-server-sdk-ai = { path = "../core", develop = true } +langchain = ">=0.3.0,<2.0" +langchain-core = ">=0.3.0,<2.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=2.8" +pytest-cov = ">=2.4.0" +pytest-asyncio = ">=0.21.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + diff --git a/packages/langchain/tests/__init__.py b/packages/langchain/tests/__init__.py new file mode 100644 index 0000000..8a69638 --- /dev/null +++ b/packages/langchain/tests/__init__.py @@ -0,0 +1,2 @@ +"""Tests for LaunchDarkly Server SDK for AI - LangChain provider.""" + diff --git a/packages/langchain/tests/test_langchain_provider.py b/packages/langchain/tests/test_langchain_provider.py new file mode 100644 index 0000000..db1913f --- /dev/null +++ b/packages/langchain/tests/test_langchain_provider.py @@ -0,0 +1,222 @@ +"""Tests for LangChain provider implementation.""" + +import pytest +from unittest.mock import AsyncMock, Mock + +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage + +from ldai.models import LDMessage +from ldai.providers.langchain import LangChainProvider +from ldai.tracker import TokenUsage + + +class TestMessageConversion: + """Test conversion between LD messages and LangChain messages.""" + + def test_convert_multiple_messages(self): + """Test converting a conversation with all message types.""" + ld_messages = [ + LDMessage(role='system', content='You are helpful'), + LDMessage(role='user', content='Hello'), + LDMessage(role='assistant', content='Hi there!'), + ] + lc_messages = LangChainProvider.convert_messages_to_langchain(ld_messages) + + assert len(lc_messages) == 3 + assert isinstance(lc_messages[0], SystemMessage) + assert isinstance(lc_messages[1], HumanMessage) + assert isinstance(lc_messages[2], AIMessage) + assert lc_messages[0].content == 'You are helpful' + assert lc_messages[1].content == 'Hello' + assert lc_messages[2].content == 'Hi there!' + + def test_convert_unsupported_role_raises_error(self): + """Test that unsupported message roles raise ValueError.""" + ld_messages = [LDMessage(role='function', content='Function result')] + + with pytest.raises(ValueError, match='Unsupported message role: function'): + LangChainProvider.convert_messages_to_langchain(ld_messages) + + +class TestMetricsExtraction: + """Test metrics extraction from LangChain response metadata.""" + + def test_extract_metrics_with_token_usage(self): + """Test extracting token usage from response metadata.""" + response = AIMessage( + content='Hello, world!', + response_metadata={ + 'token_usage': { + 'total_tokens': 100, + 'prompt_tokens': 60, + 'completion_tokens': 40, + } + } + ) + + metrics = LangChainProvider.get_ai_metrics_from_response(response) + + assert metrics.success is True + assert metrics.usage is not None + assert metrics.usage.total == 100 + assert metrics.usage.input == 60 + assert metrics.usage.output == 40 + + def test_extract_metrics_with_camel_case_token_usage(self): + """Test extracting token usage with camelCase keys (some providers use this).""" + response = AIMessage( + content='Hello, world!', + response_metadata={ + 'token_usage': { + 'totalTokens': 150, + 'promptTokens': 90, + 'completionTokens': 60, + } + } + ) + + metrics = LangChainProvider.get_ai_metrics_from_response(response) + + assert metrics.success is True + assert metrics.usage is not None + assert metrics.usage.total == 150 + assert metrics.usage.input == 90 + assert metrics.usage.output == 60 + + def test_extract_metrics_without_token_usage(self): + """Test metrics extraction when no token usage is available.""" + response = AIMessage(content='Hello, world!') + + metrics = LangChainProvider.get_ai_metrics_from_response(response) + + assert metrics.success is True + assert metrics.usage is None + + +class TestInvokeModel: + """Test model invocation with LangChain provider.""" + + @pytest.mark.asyncio + async def test_invoke_model_success(self): + """Test successful model invocation.""" + mock_llm = AsyncMock() + mock_response = AIMessage( + content='Hello, user!', + response_metadata={ + 'token_usage': { + 'total_tokens': 20, + 'prompt_tokens': 10, + 'completion_tokens': 10, + } + } + ) + mock_llm.ainvoke.return_value = mock_response + + provider = LangChainProvider(mock_llm) + messages = [LDMessage(role='user', content='Hello')] + + response = await provider.invoke_model(messages) + + assert response.message.role == 'assistant' + assert response.message.content == 'Hello, user!' + assert response.metrics.success is True + assert response.metrics.usage is not None + assert response.metrics.usage.total == 20 + + @pytest.mark.asyncio + async def test_invoke_model_with_multimodal_content_warning(self): + """Test that non-string content triggers warning and marks as failure.""" + mock_llm = AsyncMock() + mock_response = AIMessage( + content=['text', {'type': 'image'}], # Non-string content + response_metadata={'token_usage': {'total_tokens': 20}} + ) + mock_llm.ainvoke.return_value = mock_response + + provider = LangChainProvider(mock_llm) + messages = [LDMessage(role='user', content='Describe this image')] + + response = await provider.invoke_model(messages) + + # Should mark as failure due to multimodal content not being supported + assert response.metrics.success is False + assert response.message.content == '' + + @pytest.mark.asyncio + async def test_invoke_model_with_exception(self): + """Test model invocation handles exceptions gracefully.""" + mock_llm = AsyncMock() + mock_llm.ainvoke.side_effect = Exception('Model API error') + + provider = LangChainProvider(mock_llm) + messages = [LDMessage(role='user', content='Hello')] + + response = await provider.invoke_model(messages) + + # Should return failure response + assert response.message.role == 'assistant' + assert response.message.content == '' + assert response.metrics.success is False + assert response.metrics.usage is None + + +class TestInvokeStructuredModel: + """Test structured output invocation.""" + + @pytest.mark.asyncio + async def test_invoke_structured_model_with_support(self): + """Test structured output when model supports with_structured_output.""" + mock_llm = Mock() + mock_structured_llm = AsyncMock() + mock_structured_llm.ainvoke.return_value = { + 'answer': 'Paris', + 'confidence': 0.95 + } + mock_llm.with_structured_output.return_value = mock_structured_llm + + provider = LangChainProvider(mock_llm) + messages = [LDMessage(role='user', content='What is the capital of France?')] + schema = {'answer': 'string', 'confidence': 'number'} + + response = await provider.invoke_structured_model(messages, schema) + + assert response.data == {'answer': 'Paris', 'confidence': 0.95} + assert response.metrics.success is True + mock_llm.with_structured_output.assert_called_once_with(schema) + + @pytest.mark.asyncio + async def test_invoke_structured_model_without_support_json_fallback(self): + """Test structured output fallback to JSON parsing when not supported.""" + mock_llm = AsyncMock() + # Model doesn't have with_structured_output + delattr(mock_llm, 'with_structured_output') if hasattr(mock_llm, 'with_structured_output') else None + + mock_response = AIMessage(content='{"answer": "Berlin", "confidence": 0.9}') + mock_llm.ainvoke.return_value = mock_response + + provider = LangChainProvider(mock_llm) + messages = [LDMessage(role='user', content='What is the capital of Germany?')] + schema = {'answer': 'string', 'confidence': 'number'} + + response = await provider.invoke_structured_model(messages, schema) + + assert response.data == {'answer': 'Berlin', 'confidence': 0.9} + assert response.metrics.success is True + + @pytest.mark.asyncio + async def test_invoke_structured_model_with_exception(self): + """Test structured output handles exceptions gracefully.""" + mock_llm = Mock() + mock_llm.with_structured_output.side_effect = Exception('Structured output error') + + provider = LangChainProvider(mock_llm) + messages = [LDMessage(role='user', content='Question')] + schema = {'answer': 'string'} + + response = await provider.invoke_structured_model(messages, schema) + + # Should return failure response + assert response.data == {} + assert response.raw_response == '' + assert response.metrics.success is False + diff --git a/release-please-config.json b/release-please-config.json index 78df6d7..583bb8f 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -1,12 +1,25 @@ { + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", "packages": { - ".": { + "packages/core": { "release-type": "python", + "package-name": "launchdarkly-server-sdk-ai", "versioning": "default", "bump-minor-pre-major": true, "include-v-in-tag": false, - "extra-files": ["ldai/__init__.py", "PROVENANCE.md"], - "include-component-in-tag": false + "extra-files": ["packages/core/ldai/__init__.py", "PROVENANCE.md"], + "include-component-in-tag": true, + "component": "core" + }, + "packages/langchain": { + "release-type": "python", + "package-name": "launchdarkly-server-sdk-ai-langchain", + "versioning": "default", + "bump-minor-pre-major": true, + "include-v-in-tag": false, + "extra-files": ["PROVENANCE.md"], + "include-component-in-tag": true, + "component": "langchain" } } }