feat: [REL-10772] Implement Langchain provider for online evals

.github/actions/build/action.yml

@@ -1,5 +1,10 @@
 name: Build distribution files
 description: 'Build distribution files'
+inputs:
+  package-path:
+    description: 'Path to the package to build'
+    required: false
+    default: '.'
 outputs:
   package-hashes:
     description: "base64-encoded sha256 hashes of distribution files"
@@ -10,10 +15,11 @@ runs:
   steps:
     - name: Build distribution files
       shell: bash
+      working-directory: ${{ inputs.package-path }}
       run: poetry build
     - name: Hash build files for provenance
       id: package-hashes
       shell: bash
-      working-directory: ./dist
+      working-directory: ${{ inputs.package-path }}/dist
       run: |
         echo "package-hashes=$(sha256sum * | base64 -w0)" >> "$GITHUB_OUTPUT"

.github/workflows/ci.yml

@@ -28,9 +28,24 @@ jobs:
       - name: Install poetry
         uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
 
-      - uses: ./.github/actions/build
+      - name: Install packages
+        run: make install
+
+      - name: Build core package
+        uses: ./.github/actions/build
+        with:
+          package-path: packages/core
+
+      - name: Build langchain package
+        uses: ./.github/actions/build
+        with:
+          package-path: packages/langchain
+
       - uses: ./.github/actions/build-docs
 
+      - name: Reinstall packages after build
+        run: make install
+
       - name: Run tests
         run: make test
 
@@ -58,8 +73,8 @@ jobs:
       - name: Install poetry
         uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
 
-      - name: Install requirements
-        run: poetry install
+      - name: Install packages
+        run: make install
 
       - name: Run tests
         run: make test

.github/workflows/manual-publish.yml

@@ -2,20 +2,26 @@ name: Publish Package
 on:
   workflow_dispatch:
     inputs:
+      package:
+        description: 'Which package to publish'
+        required: true
+        type: choice
+        options:
+          - core
+          - langchain
+          - both
       dry_run:
         description: 'Is this a dry run? If so no package will be published.'
         type: boolean
         required: true
 
 jobs:
-  build-publish:
+  publish-core:
+    if: ${{ inputs.package == 'core' || inputs.package == 'both' }}
     runs-on: ubuntu-latest
-    # Needed to get tokens during publishing.
     permissions:
       id-token: write
       contents: read
-    outputs:
-      package-hashes: ${{ steps.build.outputs.package-hashes}}
     steps:
       - uses: actions/checkout@v4
 
@@ -34,20 +40,49 @@ jobs:
 
       - uses: ./.github/actions/build
         id: build
+        with:
+          package-path: packages/core
 
-      - name: Publish package distributions to PyPI
+      - name: Publish core package to PyPI
         if: ${{ inputs.dry_run == false }}
-        uses: pypa/gh-action-pypi-publish@release/v1
+        # https://github.com/pypa/gh-action-pypi-publish/releases/tag/v1.8.13
+        uses: pypa/gh-action-pypi-publish@3cc2c35166dfc1e5ea3bb0491ffdeedcaa50d7c
         with:
-          password: ${{env.PYPI_AUTH_TOKEN}}
+          password: ${{ env.PYPI_AUTH_TOKEN }}
+          packages-dir: packages/core/dist/
 
-  release-provenance:
-    needs: [ 'build-publish' ]
+  publish-langchain:
+    if: ${{ inputs.package == 'langchain' || inputs.package == 'both' }}
+    runs-on: ubuntu-latest
     permissions:
-      actions: read
       id-token: write
-      contents: write
-    uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected]
-    with:
-      base64-subjects: "${{ needs.build-publish.outputs.package-hashes }}"
-      upload-assets: ${{ !inputs.dry_run }}
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Install poetry
+        uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
+
+      - uses: launchdarkly/gh-actions/actions/[email protected]
+        name: 'Get PyPI token'
+        with:
+          aws_assume_role: ${{ vars.AWS_ROLE_ARN }}
+          ssm_parameter_pairs: '/production/common/releasing/pypi/token = PYPI_AUTH_TOKEN'
+
+      - uses: ./.github/actions/build
+        id: build
+        with:
+          package-path: packages/langchain
+
+      - name: Publish langchain package to PyPI
+        if: ${{ inputs.dry_run == false }}
+        # Pinned to v1.8.13 (2024-06-14) for security
+        # https://github.com/pypa/gh-action-pypi-publish/releases/tag/v1.8.13
+        uses: pypa/gh-action-pypi-publish@3cc2c35166dfc1e5ea3bb0491ffdeedcaa50d7c
+        with:
+          password: ${{ env.PYPI_AUTH_TOKEN }}
+          packages-dir: packages/langchain/dist/

.github/workflows/release-please.yml

@@ -5,63 +5,91 @@ on:
     branches: [ main ]
 
 jobs:
-  release-package:
+  release-please:
     runs-on: ubuntu-latest
     permissions:
-      id-token: write # Needed if using OIDC to get release secrets.
-      contents: write # Contents and pull-requests are for release-please to make releases.
+      contents: write
       pull-requests: write
     outputs:
-      release-created: ${{ steps.release.outputs.release_created }}
-      upload-tag-name: ${{ steps.release.outputs.tag_name }}
-      package-hashes: ${{ steps.build.outputs.package-hashes}}
+      releases_created: ${{ steps.release.outputs.releases_created }}
+      core_release_created: ${{ steps.release.outputs['packages/core--release_created'] }}
+      langchain_release_created: ${{ steps.release.outputs['packages/langchain--release_created'] }}
     steps:
       - uses: googleapis/release-please-action@v4
         id: release
+        with:
+          config-file: release-please-config.json
+          manifest-file: .release-please-manifest.json
 
+  release-core:
+    needs: release-please
+    if: ${{ needs.release-please.outputs.core_release_created == 'true' }}
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    steps:
       - uses: actions/checkout@v4
-        if: ${{ steps.release.outputs.releases_created == 'true' }}
-        with:
-          fetch-depth: 0 # If you only need the current version keep this.
 
       - uses: actions/setup-python@v5
-        if: ${{ steps.release.outputs.releases_created == 'true' }}
         with:
           python-version: 3.9
 
       - name: Install poetry
-        if: ${{ steps.release.outputs.releases_created == 'true' }}
         uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
 
       - uses: launchdarkly/gh-actions/actions/[email protected]
-        if: ${{ steps.release.outputs.releases_created == 'true' }}
         name: 'Get PyPI token'
         with:
           aws_assume_role: ${{ vars.AWS_ROLE_ARN }}
           ssm_parameter_pairs: '/production/common/releasing/pypi/token = PYPI_AUTH_TOKEN'
 
       - uses: ./.github/actions/build
         id: build
-        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        with:
+          package-path: packages/core
 
       - uses: ./.github/actions/build-docs
-        if: ${{ steps.release.outputs.releases_created == 'true' }}
 
-      - name: Publish package distributions to PyPI
-        if: ${{ steps.release.outputs.releases_created == 'true' }}
-        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Publish core package to PyPI
+        # https://github.com/pypa/gh-action-pypi-publish/releases/tag/v1.8.13
+        uses: pypa/gh-action-pypi-publish@3cc2c35166dfc1e5ea3bb0491ffdeedcaa50d7c
         with:
-          password: ${{env.PYPI_AUTH_TOKEN}}
+          password: ${{ env.PYPI_AUTH_TOKEN }}
+          packages-dir: packages/core/dist/
 
-  release-provenance:
-    needs: [ 'release-package' ]
-    if: ${{ needs.release-package.outputs.release-created == 'true' }}
+  release-langchain:
+    needs: release-please
+    if: ${{ needs.release-please.outputs.langchain_release_created == 'true' }}
+    runs-on: ubuntu-latest
     permissions:
-      actions: read
       id-token: write
       contents: write
-    uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected]
-    with:
-      base64-subjects: "${{ needs.release-package.outputs.package-hashes }}"
-      upload-assets: true
-      upload-tag-name: ${{ needs.release-package.outputs.upload-tag-name }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Install poetry
+        uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
+
+      - uses: launchdarkly/gh-actions/actions/[email protected]
+        name: 'Get PyPI token'
+        with:
+          aws_assume_role: ${{ vars.AWS_ROLE_ARN }}
+          ssm_parameter_pairs: '/production/common/releasing/pypi/token = PYPI_AUTH_TOKEN'
+
+      - uses: ./.github/actions/build
+        id: build
+        with:
+          package-path: packages/langchain
+
+      - name: Publish langchain package to PyPI
+        # Pinned to v1.8.13 (2024-06-14) for security
+        # https://github.com/pypa/gh-action-pypi-publish/releases/tag/v1.8.13
+        uses: pypa/gh-action-pypi-publish@3cc2c35166dfc1e5ea3bb0491ffdeedcaa50d7c
+        with:
+          password: ${{ env.PYPI_AUTH_TOKEN }}
+          packages-dir: packages/langchain/dist/

.release-please-manifest.json

@@ -1,3 +1,4 @@
 {
-  ".": "0.10.1"
+  "packages/core": "0.10.1",
+  "packages/langchain": "0.1.0"
 }

Makefile

@@ -14,31 +14,66 @@ help: #! Show this help message
 	@grep -h -F '#!' $(MAKEFILE_LIST) | grep -v grep | sed 's/:.*#!/:/' | column -t -s":"
 
 .PHONY: install
-install:
-	@poetry install
+install: #! Install all packages
+	@echo "Installing core package..."
+	@cd packages/core && poetry install
+	@echo "Installing langchain package..."
+	@cd packages/langchain && poetry install
+
+.PHONY: install-core
+install-core: #! Install core package only
+	@cd packages/core && poetry install
+
+.PHONY: install-langchain
+install-langchain: #! Install langchain package only
+	@cd packages/langchain && poetry install
 
 #
 # Quality control checks
 #
 
 .PHONY: test
-test: #! Run unit tests
-test: install
-	@poetry run pytest $(PYTEST_FLAGS)
+test: #! Run unit tests for all packages
+	@echo "Testing core package..."
+	@cd packages/core && poetry run pytest $(PYTEST_FLAGS)
+	@echo "Testing langchain package..."
+	@cd packages/langchain && poetry run pytest $(PYTEST_FLAGS)
+
+.PHONY: test-core
+test-core: #! Run unit tests for core package
+	@cd packages/core && poetry run pytest $(PYTEST_FLAGS)
+
+.PHONY: test-langchain
+test-langchain: #! Run unit tests for langchain package
+	@cd packages/langchain && poetry run pytest $(PYTEST_FLAGS)
 
 .PHONY: lint
 lint: #! Run type analysis and linting checks
-lint: install
-	@poetry run mypy ldai
-	@poetry run isort --check --atomic ldai
-	@poetry run pycodestyle ldai
+	@echo "Linting core package..."
+	@cd packages/core && poetry run mypy ldai
+	@cd packages/core && poetry run isort --check --atomic ldai
+	@cd packages/core && poetry run pycodestyle ldai
+
+.PHONY: build
+build: #! Build all packages
+	@echo "Building core package..."
+	@cd packages/core && poetry build
+	@echo "Building langchain package..."
+	@cd packages/langchain && poetry build
+
+.PHONY: build-core
+build-core: #! Build core package
+	@cd packages/core && poetry build
+
+.PHONY: build-langchain
+build-langchain: #! Build langchain package
+	@cd packages/langchain && poetry build
 
 #
 # Documentation generation
 #
 
 .PHONY: docs
 docs: #! Generate sphinx-based documentation
-	@poetry install --with docs
-	@cd docs
-	@poetry run $(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@cd packages/core && poetry install --with docs
+	@cd packages/core && poetry run $(SPHINXBUILD) -M html "../../$(SOURCEDIR)" "../../$(BUILDDIR)" $(SPHINXOPTS) $(O)