End-to-end (integration testing) for Python wheels & OpenTDF Platform

.github/workflows/build-golang-macos.yaml

@@ -76,32 +76,20 @@ jobs:
         # FIXME: Add more caching
         - name: Add gopy dependencies and build wheel
           run: |
-              # Since we don't have our wheel build / install configured yet we use '--no-root'
-              poetry install --no-root
-
-              source $(poetry env info --path)/bin/activate
-
-              # Add Go bin directory to PATH
-              echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV
-
-              go install golang.org/x/tools/cmd/goimports@latest
-
-              go install github.com/go-python/gopy@latest
-
-              poetry run pip install --upgrade setuptools wheel
-
-              gopy build --output=otdf_python -vm=python3 .
-
-              poetry run python3 setup.py bdist_wheel
+            ./ci-build.sh
 
         - name: Test Python wheel
           run: |
               # Test wheel installation
-              pip install dist/otdf_python-0.0.15-py3-none-any.whl
+              pip install dist/otdf_python-0.1.1-py3-none-any.whl
 
               # Test wheel functionality
               # python3 validate_otdf_python.py
 
+        - uses: ./.github/workflows/platform-integration-test.yaml
+          with:
+            wheel: dist/otdf_python-0.1.1-py3-none-any.whl
+
   # release:
   #   needs: build
   #   runs-on: macos-latest

.github/workflows/build-golang-ubuntu.yaml

@@ -57,78 +57,36 @@ jobs:
             # FIXME: Add more caching
             - name: Add gopy dependencies and build wheel
               run: |
-                  # Since we don't have our wheel build / install configured yet we use '--no-root'
-                  poetry install --no-root
-
-                  source $(poetry env info --path)/bin/activate
-
-                  # Add Go bin directory to PATH
-                  echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV
-
-                  go install golang.org/x/tools/cmd/goimports@latest
-
-                  go install github.com/go-python/gopy@latest
-
-                  poetry run pip install --upgrade setuptools wheel
-
-                  gopy build --output=otdf_python -vm=python3 .
-
-                  poetry run python3 setup.py bdist_wheel
-
-    # TODO: Can this be a distinct job?
-    # exercise-ssh:
-    #     needs: build
-
-    #     runs-on: ubuntu-latest
-
-    #     steps:
-    #        - uses: actions/checkout@v4
+                ./ci-build.sh
 
             - name: Check existing workspace content
               run: |
                   ls -la
                   ls -la dist/
 
-            # - name: Set up Docker Buildx
-            #   uses: docker/setup-buildx-action@v3
-
-            # - name: Install Vagrant
-            #   run: sudo apt-get update && sudo apt-get install -y vagrant
-
-            # - name: Create Vagrantfile
-            #   run: |
-            #       cat << EOF > Vagrantfile
-            #       Vagrant.configure("2") do |config|
-            #         config.vm.provider "docker" do |d|
-            #           d.image = "ubuntu"
-            #           d.cmd = [ "/bin/bash", "-c", "echo 'Hello world'" ]
-            #           d.remains_running = false
-            #         end
-            #       end
-            #       EOF
-
-            # - name: Run vagrant up
-            #   run: vagrant up --provider=docker
-
-            # - name: Launch the Docker/Vagrant test machine
-            #   run: |
-            #       # Change directory
-            #       cd ssh-servers/docker-in-vagrant
-
-            #       # List content in the directory
-            #       ls -la
-
-            #       # Launch machine
-            #       vagrant up --provider=docker
-
-            #       # Connect to it with SSH
-            #       vagrant ssh -c "echo 'Hello Ubuntu'"
-
             - name: Test Python wheel
               run: |
                   # Test wheel installation
-                  pip install dist/otdf_python-0.0.15-py3-none-any.whl
+                  pip install dist/otdf_python-0.1.1-py3-none-any.whl
 
                   # DISABLED: Need to figure out Ubuntu nested VM
                   # Test wheel functionality
                   # python3 validate_otdf_python.py
+
+            - uses: actions/cache/restore@v4
+              with:
+                path: dist/otdf_python-0.1.1-py3-none-any.whl
+                key: ${{ runner.os }}-data-${{ github.sha }}
+
+            - uses: actions/cache/save@v4
+              with:
+                path: dist/otdf_python-0.1.1-py3-none-any.whl
+                key: ${{ runner.os }}-data-${{ github.sha }}
+                restore-keys: |
+                  ${{ runner.os }}-data-
+
+    integration-test:
+      needs: build
+      uses: ./.github/workflows/platform-integration-test.yaml
+      with:
+        wheel: dist/otdf_python-0.1.1-py3-none-any.whl

.github/workflows/platform-integration-test.yaml

@@ -1,31 +1,43 @@
 # Based on
 #   https://github.com/opentdf/java-sdk/blob/v0.6.1/.github/workflows/checks.yaml
+#
+# Except, that this is a "Composite Action", and specifies 'shell: bash' for
+# each 'run:' step.
 name: "Platform Integration testing"
 
 on:
-  pull_request:
-
-# on:
-#   pull_request:
-#     branches:
-#       - main
-#   push:
-#     branches:
-#       - main
+  workflow_call:
+    inputs:
+      wheel:
+        description: The Python wheel to test
+        required: true
+        type: string
 
 permissions:
   contents: read
 
 jobs:
-
-  platform-integration:
-    runs-on: ubuntu-22.04
+  integration_test:
+    runs-on: ubuntu-latest
     steps:
-      - name: Checkout Java SDK
+      - name: Checkout this repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-      - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa
+
+      - uses: actions/cache/restore@v4
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          path: dist/otdf_python-0.1.1-py3-none-any.whl
+          key: ${{ runner.os }}-data-${{ github.sha }}
+
+      - name: Prove that the input file is available
+        shell: bash
+        run: |
+          ls -la
+          ls -la "${{ inputs.wheel }}"
+          du -sh "${{ inputs.wheel }}"
+
+      # - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Check out platform
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
@@ -44,10 +56,13 @@ jobs:
             platform/protocol/go/go.sum
             platform/sdk/go.sum
       - run: go mod download
+        shell: bash
         working-directory: platform
       - run: go mod verify
+        shell: bash
         working-directory: platform
       - name: Create keys
+        shell: bash
         run: |
           .github/scripts/init-temp-keys.sh
           cp opentdf-dev.yaml opentdf.yaml
@@ -64,12 +79,15 @@ jobs:
       #       -alias localhost-for-tests
       #   working-directory: platform/keys
       - name: Bring the services up
+        shell: bash
         run: docker compose up -d --wait --wait-timeout 240
         working-directory: platform
       - name: Provision keycloak
+        shell: bash
         run: go run ./service provision keycloak
         working-directory: platform
       - name: Provision fixtures
+        shell: bash
         run: go run ./service provision fixtures
         working-directory: platform
       - name: Start server in background
@@ -83,12 +101,37 @@ jobs:
           wait-for: 90s
           working-directory: platform
       - name: Get grpcurl
+        shell: bash
         run: go install github.com/fullstorydev/grpcurl/cmd/[email protected]
       - name: Make sure that the platform is up
+        shell: bash
         run: |
           grpcurl -plaintext localhost:8080 list && \
           grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+            python-version: '3.11'
+
+      - name: Validate the Python SDK
+        env:
+          OPENTDF_CLIENT_ID: "opentdf-sdk"
+          OPENTDF_CLIENT_SECRET: "secret"
+          OPENTDF_HOSTNAME: "localhost:8080"
+          OIDC_TOKEN_ENDPOINT: "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token"
+          OPENTDF_KAS_URL: "http://localhost:8080/kas"
+        run: |
+          mkdir validation
+          wheel="$(basename ${{ inputs.wheel }} )"
+          cp -v "${{ inputs.wheel }}" validation/
+          cp -v validate_otdf_python.py validation/
+          cd validation
+          python -m venv .venv
+          source .venv/bin/activate
+          pip install ./"$wheel"
+          python validate_otdf_python.py
+
       # - name: Validate the SDK through the command line interface
       #   run: |
       #     printf 'here is some data to encrypt' > data

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ exclude: |
 # See https://pre-commit.com/hooks.html for more hooks#
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.6.0
+      rev: v5.0.0
       hooks:
           - id: check-yaml
           - id: end-of-file-fixer
@@ -40,7 +40,7 @@ repos:
 
     - repo: https://github.com/astral-sh/ruff-pre-commit
       # Ruff version.
-      rev: v0.6.9
+      rev: v0.7.4
       hooks:
         # Run the linter.
         - id: ruff

ci-build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -eou pipefail
+
+# Since we don't have our wheel build / install configured yet we use '--no-root'
+poetry install --no-root
+
+source $(poetry env info --path)/bin/activate
+
+# Add Go bin directory to PATH
+echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV
+
+go install golang.org/x/tools/cmd/goimports@latest
+
+go install github.com/go-python/gopy@latest
+
+poetry run pip install --upgrade setuptools wheel
+
+gopy build --output=otdf_python -vm=python3 .
+
+poetry run python3 setup.py bdist_wheel

go.mod

@@ -1,39 +1,39 @@
 module gotdf_python
 
-go 1.22.0
+go 1.22.7
 
-toolchain go1.23.2
+toolchain go1.23.3
 
-require github.com/opentdf/platform/sdk v0.3.13
+require github.com/opentdf/platform/sdk v0.3.21
 
 require (
-	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240920164238-5a7b106cbb87.2 // indirect
+	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.2-20240920164238-5a7b106cbb87.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gowebpki/jcs v1.0.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/httprc v1.0.6 // indirect
 	github.com/lestrrat-go/iter v1.0.2 // indirect
-	github.com/lestrrat-go/jwx/v2 v2.1.1 // indirect
+	github.com/lestrrat-go/jwx/v2 v2.1.2 // indirect
 	github.com/lestrrat-go/option v1.0.1 // indirect
-	github.com/opentdf/platform/lib/ocrypto v0.1.6 // indirect
-	github.com/opentdf/platform/protocol/go v0.2.17 // indirect
+	github.com/opentdf/platform/lib/ocrypto v0.1.7 // indirect
+	github.com/opentdf/platform/protocol/go v0.2.22 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/crypto v0.27.0 // indirect
-	golang.org/x/net v0.29.0 // indirect
-	golang.org/x/oauth2 v0.23.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
-	golang.org/x/text v0.18.0 // indirect
-	golang.org/x/tools v0.25.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240930140551-af27646dc61f // indirect
-	google.golang.org/grpc v1.67.1 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	golang.org/x/crypto v0.29.0 // indirect
+	golang.org/x/net v0.31.0 // indirect
+	golang.org/x/oauth2 v0.24.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/text v0.20.0 // indirect
+	golang.org/x/tools v0.27.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect
+	google.golang.org/grpc v1.68.0 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 )