diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml new file mode 100644 index 000000000..85a0b5116 --- /dev/null +++ b/.github/workflows/build-bake-preview.yaml @@ -0,0 +1,99 @@ +on: + schedule: + - cron: '0 8 * * *' + - cron: '0 9 * * *' + push: + branches: + - main + - dev + - dev-rspm + pull_request: + +name: Preview - Build, Test, and Push +jobs: + build-preview: + runs-on: ubuntu-latest-4x + + concurrency: + group: bake-preview-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Check Out main Branch + if: github.event.schedule == '0 8 * * *' + uses: actions/checkout@v3 + with: + ref: 'main' + + - name: Check Out Repo at Triggered Branch + if: github.event.schedule != '0 8 * * *' + uses: actions/checkout@v3 + + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config: ./share/buildkitd.toml + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Python dependencies + run: | + pip install requests + + - name: Get Version + id: get-version + run: | + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) + echo "WORKBENCH_DAILY_VERSION=$WORKBENCH_DAILY_VERSION" >> $GITHUB_OUTPUT + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) + echo "WORKBENCH_PREVIEW_VERSION=$WORKBENCH_PREVIEW_VERSION" >> $GITHUB_OUTPUT + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) + echo "PACKAGE_MANAGER_DAILY_VERSION=$PACKAGE_MANAGER_DAILY_VERSION" >> $GITHUB_OUTPUT + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) + echo "CONNECT_DAILY_VERSION=$CONNECT_DAILY_VERSION" >> $GITHUB_OUTPUT + + - name: Build and test + id: build + uses: docker/bake-action@v4 + env: + WORKBENCH_DAILY_VERSION: ${{ steps.get-version.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ steps.get-version.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ steps.get-version.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ steps.get-version.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd + targets: "build-test" + push: false + files: docker-bake.preview.hcl + + - name: Test Connect (privileged container) + run: | + just -f build.justfile test-connect-preview + + - name: Push + id: push + if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + uses: docker/bake-action@v4 + env: + WORKBENCH_DAILY_VERSION: ${{ steps.get-version.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ steps.get-version.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ steps.get-version.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ steps.get-version.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd + targets: "build" + push: true + files: docker-bake.preview.hcl diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml new file mode 100644 index 000000000..c78a7d030 --- /dev/null +++ b/.github/workflows/build-bake.yaml @@ -0,0 +1,96 @@ +on: + push: + branches: + - main + - dev + pull_request: + +name: Release - Build, Test, and Push +jobs: + build: + name: Build and Test images + runs-on: ubuntu-latest-4x + + concurrency: + group: bake-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build and test + id: build + uses: docker/bake-action@v4 + with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd + targets: "build-test" + push: false + + - name: Test Connect (privileged container) + run: | + just -f build.justfile test-connect + + - name: Push + id: push + if: github.ref == 'refs/heads/main' + uses: docker/bake-action@v4 + with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd + targets: "build" + push: true + + build-content: + runs-on: ubuntu-latest-4x + + concurrency: + group: bake-content-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config: ./share/buildkitd.toml + + - name: Build + id: build + uses: docker/bake-action@v4 + with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd + workdir: content/. + push: false + + - name: Push + id: push + if: github.ref == 'refs/heads/main' + uses: docker/bake-action@v4 + with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd + workdir: content/. + push: true diff --git a/.github/workflows/build-content.yaml b/.github/workflows/build-content.yaml deleted file mode 100644 index 4b7db4e1c..000000000 --- a/.github/workflows/build-content.yaml +++ /dev/null @@ -1,215 +0,0 @@ -on: - push: - branches: - - main - - dev - pull_request: - -name: Content Images - Build, Test, Scan, and Push -jobs: - matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - - id: set-matrix - run: | - MATRIX=$(jq -Mcr < content/matrix.json) - echo "matrix=$MATRIX" >> $GITHUB_OUTPUT - - build: - runs-on: ubuntu-latest - needs: matrix - name: content-base-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}--${{ github.ref }} - - permissions: - contents: read - packages: write - - concurrency: - group: content-base-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ github.ref }} - cancel-in-progress: true - - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.matrix.outputs.matrix) }} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - get-content-args \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - get-content-tags \ - content-base \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ${{ matrix.config.os }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push content base image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./content/base - os: ${{ matrix.config.os }} - product: content-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./content/base - os: ${{ matrix.config.os }} - product: content-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic - - build-pro: - runs-on: ubuntu-latest - needs: [ matrix, build ] - name: content-pro-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}--${{ github.ref }} - concurrency: - group: content-pro-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ github.ref }} - cancel-in-progress: true - - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.matrix.outputs.matrix) }} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - get-content-args \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ${{ matrix.config.drivers }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - get-content-tags \ - content-pro \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ${{ matrix.config.os }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push content pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./content/pro - os: ${{ matrix.config.os }} - product: content-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./content/pro - os: ${{ matrix.config.os }} - product: content-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml deleted file mode 100644 index 470470bb9..000000000 --- a/.github/workflows/build-prerelease.yaml +++ /dev/null @@ -1,144 +0,0 @@ -on: - schedule: - # every morning at 8am UTC - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule - # make sure to change the `if` commands below when changing the schedule - - cron: '0 8 * * *' - - cron: '0 9 * * *' - push: - branches: - - main - - dev - - dev-rspm - pull_request: - -name: Prerelease - Build, Test, Scan, and Push -jobs: - build: - runs-on: ubuntu-latest - name: build-${{ matrix.config.type }}-${{ matrix.config.product }}-${{ matrix.config.os }} - - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: - config: - - {product: "workbench", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "workbench", type: "preview", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "connect", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "connect-content-init", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "package-manager", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - {product: "r-session-complete", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "r-session-complete", type: "daily", os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: "r-session-complete", type: "preview", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "r-session-complete", type: "preview", os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - concurrency: - group: prerelease-build-${{ matrix.config.type }}-${{ matrix.config.product }}-${{ matrix.config.os }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Check Out main Branch - if: github.event.schedule == '0 8 * * *' - uses: actions/checkout@v3 - with: - ref: 'main' - - - name: Check Out Repo at Triggered Branch - if: github.event.schedule != '0 8 * * *' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=$(just -f ci.Justfile get-version ${{ matrix.config.product }} --type=${{ matrix.config.type }} --local) - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-prerelease-args \ - ${{ matrix.config.type }} \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-prerelease-tags \ - ${{ matrix.config.type }} \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: ${{ matrix.config.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: product-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml deleted file mode 100644 index 72eb2cc5b..000000000 --- a/.github/workflows/build-release.yaml +++ /dev/null @@ -1,360 +0,0 @@ -on: - schedule: - - cron: '0 12 * * 1' # If updating this value, be sure to update logic for all `push-image` arguments! - push: - branches: - - main - - dev - pull_request: - -name: Release - Build, Test, Scan, and Push -jobs: - build-base: - runs-on: ubuntu-latest - name: product-base-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }} - - permissions: - contents: read - packages: write - - concurrency: - group: base-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }}-${{ github.ref }} - cancel-in-progress: true - - strategy: - fail-fast: false - matrix: - config: - - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 12 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 12 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-args ${{ matrix.config.os }} product-base \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-tags ${{ matrix.config.os }} product-base \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./product/base - os: ${{ matrix.config.os }} - product: product-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./product/base - os: ${{ matrix.config.os }} - product: product-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic - - build-base-pro: - needs: build-base - runs-on: ubuntu-latest - name: product-base-pro-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }} - - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: - config: - - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - concurrency: - group: build-base-pro-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 12 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 12 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-args ${{ matrix.config.os }} product-base-pro \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-tags ${{ matrix.config.os }} product-base-pro \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./product/pro - os: ${{ matrix.config.os }} - product: product-base-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./product/pro - os: ${{ matrix.config.os }} - product: product-base-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic - - build-products: - needs: [ build-base, build-base-pro ] - runs-on: ubuntu-latest - name: build-${{ matrix.config.product }}-${{ matrix.config.os }} - - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: - config: - - {product: 'workbench', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'connect', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: 'connect-content-init', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - {product: 'r-session-complete', os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'r-session-complete', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - concurrency: - group: build-products-${{ matrix.config.product }}-${{ matrix.config.os }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 12 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 12 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=`just -f ci.Justfile get-version ${{ matrix.config.product }} --type=release --local` - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-product-args \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-product-tags \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push product image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: ${{ matrix.config.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push product image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: ${{ matrix.config.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-workbench-aml.yaml b/.github/workflows/build-workbench-aml.yaml deleted file mode 100644 index 9e70c1490..000000000 --- a/.github/workflows/build-workbench-aml.yaml +++ /dev/null @@ -1,127 +0,0 @@ -on: - schedule: - - cron: '0 14 * * 1' # If updating this value, be sure to update logic for all `push-image` arguments! - push: - branches: - - main - - dev - pull_request: - -name: Workbench for Azure ML - Build, Test, Scan, and Push -jobs: - build-workbench-for-azure-ml: - env: - product: workbench-for-microsoft-azure-ml - os: ubuntu2204 - r-primary: 4.2.3 - r-alternate: 4.1.3 - py-primary: 3.9.14 - py-alternate: 3.8.15 - runs-on: ubuntu-latest-4x - name: build-workbench-for-azure-ml - - permissions: - contents: read - packages: write - - steps: - - - name: Check Out Repo - cron main - if: github.event.schedule == '0 14 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 14 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=$(just -f ci.Justfile get-version ${{ env.product }} --type=release --local) - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ env.r-primary }} \ - R_VERSION_ALT=${{ env.r-alternate }} \ - PYTHON_VERSION=${{ env.py-primary }} \ - PYTHON_VERSION_ALT=${{ env.py-alternate }} \ - get-product-args \ - ${{ env.product }} \ - ${{ env.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ env.r-primary }} \ - R_VERSION_ALT=${{ env.r-alternate }} \ - PYTHON_VERSION=${{ env.py-primary }} \ - PYTHON_VERSION_ALT=${{ env.py-alternate }} \ - get-product-tags \ - ${{ env.product }} \ - ${{ env.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push product image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./${{ env.product }} - os: ${{ env.os }} - product: ${{ env.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push product image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./${{ env.product }} - os: ${{ env.os }} - product: ${{ env.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-workbench-gcw.yaml b/.github/workflows/build-workbench-gcw.yaml deleted file mode 100644 index 750ae7bd4..000000000 --- a/.github/workflows/build-workbench-gcw.yaml +++ /dev/null @@ -1,99 +0,0 @@ -on: - schedule: - - cron: '0 14 * * 1' # If updating this value, be sure to update logic for all `push-image` arguments! - push: - branches: - - main - - dev - pull_request: - -name: Workbench for GCW - Build, Test, Scan, and Push -jobs: - build-workbench-for-google-cloud-workstations: - runs-on: ubuntu-latest-4x - name: build-workbench-for-google-cloud-workstations - - permissions: - contents: read - packages: write - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 14 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 14 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=$(just workbench-for-google-cloud-workstations/get-version) - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$(just workbench-for-google-cloud-workstations/get-build-args) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$(just workbench-for-google-cloud-workstations/get-build-tags) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./workbench-for-google-cloud-workstations - os: ubuntu2004 - product: workbench-for-google-cloud-workstations - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./workbench-for-google-cloud-workstations - os: ubuntu2004 - product: workbench-for-google-cloud-workstations - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.gitignore b/.gitignore index c9c09d208..f2b8f9669 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ data .idea/ tmp*/ .python-version +.out/ +.cache/ diff --git a/Justfile b/Justfile index 591527882..3b2510765 100644 --- a/Justfile +++ b/Justfile @@ -112,13 +112,11 @@ update-rsw-versions: sed {{ sed_vars }} "s/RSW_VERSION:.*/RSW_VERSION: {{ RSW_VERSION }}/g" docker-compose.yml sed {{ sed_vars }} "s/rstudio\/rstudio-workbench:.*/rstudio\/rstudio-workbench:$(just _get-clean-version {{ RSW_VERSION }})/g" docker-compose.yml sed {{ sed_vars }} "s/^RSW_VERSION := .*/RSW_VERSION := \"{{ RSW_VERSION }}\"/g" \ - r-session-complete/Justfile \ - workbench/Justfile \ - workbench-for-microsoft-azure-ml/Justfile \ Justfile sed {{ sed_vars }} "s/[0-9]\{4\}\.[0-9]\{1,2\}\.[0-9]\{1,2\}/`just _get-clean-version {{ RSW_VERSION }}`/g" \ workbench/README.md \ r-session-complete/README.md + sed -i '/variable WORKBENCH_VERSION/!b;n;c\ \ \ \ default = "{{ RSW_VERSION }}"' docker-bake.hcl # just RSPM_VERSION=1.2.3 update-rspm-versions update-rspm-versions: @@ -133,6 +131,7 @@ update-rspm-versions: package-manager/Justfile \ Justfile sed {{ sed_vars }} -E "s/[0-9]{4}\.[0-9]{1,2}\.[0-9]{1,2}/`just _get-clean-version {{ RSPM_VERSION }}`/g" package-manager/README.md + sed -i '/variable PACKAGE_MANAGER_VERSION/!b;n;c\ \ \ \ default = "{{ RSPM_VERSION }}"' docker-bake.hcl # just RSC_VERSION=1.2.3 update-rsc-versions update-rsc-versions: @@ -151,9 +150,11 @@ update-rsc-versions: sed {{ sed_vars }} -E "s/[0-9]{4}\.[0-9]{1,2}\.[0-9]{1,2}/`just _get-clean-version {{ RSC_VERSION }}`/g" \ connect/README.md \ connect-content-init/README.md + sed -i '/variable CONNECT_VERSION/!b;n;c\ \ \ \ default = "{{ RSC_VERSION }}"' docker-bake.hcl # just R_VERSION=3.2.1 R_VERSION_ALT=4.1.0 update-r-versions -update-r-versions: +update-r-versions: update-default-r-versions +update-default-r-versions: #!/usr/bin/env bash set -euxo pipefail # Update primary R versions @@ -197,7 +198,8 @@ update-r-versions: ci.Justfile # just PYTHON_VERSION=3.9.5 PYTHON_VERSION_ALT=3.8.10 update-py-versions -update-py-versions: +update-py-versions: update-default-py-versions +update-default-py-versions: #!/usr/bin/env bash set -euxo pipefail # Update primary Python versions @@ -267,6 +269,7 @@ update-drivers-versions: r-session-complete/Justfile \ product/pro/Justfile \ ci.Justfile + sed -i '/variable DRIVERS_VERSION/!b;n;c\ \ \ \ default = "{{ RSC_VERSION }}"' docker-bake.hcl update-quarto-versions: #!/usr/bin/env bash @@ -283,20 +286,7 @@ update-quarto-versions: connect/rstudio-connect.gcfg sed {{ sed_vars }} "s/qver=\${QUARTO_VERSION:-.*}/qver=\${QUARTO_VERSION:-{{ QUARTO_VERSION }}}/g" \ content/base/maybe_install_quarto.sh - - -# just test-image preview workbench 12.0.11-8 tag1 tag2 tag3 ... -test-image $PRODUCT $VERSION +IMAGES: - #!/usr/bin/env bash - set -euxo pipefail - IMAGES="{{IMAGES}}" - read -ra IMAGE_ARRAY <<<"$IMAGES" - just \ - R_VERSION={{R_VERSION}} \ - R_VERSION_ALT={{R_VERSION_ALT}} \ - PYTHON_VERSION={{PYTHON_VERSION}} \ - PYTHON_VERSION_ALT={{PYTHON_VERSION_ALT}} \ - $PRODUCT/test "${IMAGE_ARRAY[0]}" "$VERSION" + sed -i '/variable DEFAULT_QUARTO_VERSION/!b;n;c\ \ \ \ default = "{{ QUARTO_VERSION }}"' docker-bake.hcl # just lint workbench ubuntu2204 lint $PRODUCT $OS: diff --git a/NEWS.md b/NEWS.md index 2a981260b..2265944d3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,10 @@ changed in each image. This file only captures pervasive, repository-wide changes. +# 2024-04-08 + +- Change build orchestration to use `docker buildx bake` for all images. + # 2024-03-14 - Update Professional Drivers to 2024.03.0 diff --git a/build.justfile b/build.justfile new file mode 100755 index 000000000..0ad10bea2 --- /dev/null +++ b/build.justfile @@ -0,0 +1,116 @@ +#!/usr/bin/env just --justfile + +# Targets for managing the buildx builder for Posit images + +# just create-builder +create-builder: + docker buildx create \ + --name posit-builder \ + --buildkitd-flags '--allow-insecure-entitlement security.insecure' \ + --buildkitd-config "{{justfile_directory()}}/share/local_buildkitd.toml" + +# just delete-builder +delete-builder: + docker buildx rm posit-builder + +# Build and bake + +# just bake connect +bake target: + just -f {{justfile()}} create-builder || true + docker buildx bake --builder=posit-builder -f docker-bake.hcl {{target}} + +preview-bake target branch="$(git branch --show-current)": + just -f {{justfile()}} create-builder || true + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + docker buildx bake --builder=posit-builder -f docker-bake.preview.hcl {{target}} + +# just plan +plan: + docker buildx bake -f docker-bake.hcl --print + +preview-plan branch="$(git branch --show-current)": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + docker buildx bake -f docker-bake.preview.hcl --print + +# just build +build: + just -f {{justfile()}} bake build + +preview-build branch="$(git branch --show-current)": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + just -f {{justfile()}} bake preview-build + +# just test +test: + just -f {{justfile()}} bake test + just -f {{justfile()}} test-connect ubuntu2204 + +# just preview-test +preview-test branch="$(git branch --show-current)": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + just -f {{justfile()}} bake preview-test + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + just -f {{justfile()}} test-connect-preview ubuntu2204 + +# just build-test +build-test: + just -f {{justfile()}} build + just -f {{justfile()}} test + +preview-build-test: + just -f {{justfile()}} preview-build + just -f {{justfile()}} preview-test + +# Run tests + +# just test-connect ubuntu2204 +test-connect os="ubuntu2204": + #!/bin/bash + just -f {{justfile()}} create-builder || true + build_cmd=$(python3 {{justfile_directory()}}/tools/bake_test_command_extract.py connect {{os}}) + echo $build_cmd + if [[ $? -ne 0 ]]; then + echo "Failed to extract build command" + exit 1 + fi + bash -c "$build_cmd" + +# just test-connect ubuntu2204 +test-connect-preview os="ubuntu2204": + #!/bin/bash + set -x + just -f {{justfile()}} create-builder || true + build_cmd=$( \ + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + python3 {{justfile_directory()}}/tools/bake_test_command_extract.py connect-daily {{os}} --file "docker-bake.preview.hcl" \ + ) + echo $build_cmd + if [[ $? -ne 0 ]]; then + echo "Failed to extract build command" + exit 1 + fi + bash -c "$build_cmd" diff --git a/ci.Justfile b/ci.Justfile index 392613192..06324e8ec 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -59,55 +59,6 @@ _get-rsw-download-url TYPE OS: get-version +NARGS: ./tools/get-version.py {{NARGS}} -# just get-base-args ubuntu2204 base|pro -get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: - #!/usr/bin/env bash - set -euxo pipefail - if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then - SRC_IMAGE_NAME="" - CTX_PATH="./product/base" - FILE_PATH="./product/base/Dockerfile.${OS}" - elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then - SRC_IMAGE_NAME="product-base" - CTX_PATH="./product/pro" - FILE_PATH="./product/pro/Dockerfile.${OS}" - fi - if [[ $BRANCH != "main" ]]; then - SRC_IMAGE_NAME="${SRC_IMAGE_NAME}-dev" - fi - - if [[ "${OS}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - printf "R_VERSION={{ R_VERSION }} - R_VERSION_ALT={{ R_VERSION_ALT }} - PYTHON_VERSION={{ PYTHON_VERSION }} - PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} - QUARTO_VERSION={{ QUARTO_VERSION }} - DRIVERS_VERSION=${_DRIVERS_VERSION} - SRC_IMAGE_NAME=${SRC_IMAGE_NAME}" - -# just get-base-tags ubuntu2204 base|pro -get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="" - if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then - IMAGE_NAME="product-base" - elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then - IMAGE_NAME="product-base-pro" - fi - if [[ $BRANCH != "main" ]]; then - IMAGE_NAME="${IMAGE_NAME}-dev" - fi - - echo ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}},\ - ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}},\ - ghcr.io/rstudio/${IMAGE_NAME}:${OS} - # just get-product-args connect ubuntu2204 2023.05.0 get-product-args $PRODUCT $OS $VERSION $USE_S3="false" $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`: #!/usr/bin/env bash @@ -286,20 +237,3 @@ get-prerelease-tags $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: done tags=$(IFS="," ; echo "${tag_array[*]}") echo "${tags}" - -# just get-content-args 4.2.3 3.9.17 -get-content-args r-ver py-ver drivers-ver="": - #!/usr/bin/env bash - printf "R_VERSION={{r-ver}} - PYTHON_VERSION={{py-ver}} - DRIVERS_VERSION={{drivers-ver}}" - -# just get-content-tags content-base|content-pro 4.2.3 3.9.17 ubuntu2204 -get-content-tags image-name r-ver py-ver os: - #!/usr/bin/env bash - OS=$(just _parse-os {{os}}) - OS_ALT=$(just _rev-parse-os {{os}}) - echo rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS},\ - ghcr.io/rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS},\ - rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS_ALT},\ - ghcr.io/rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS_ALT} diff --git a/connect-content-init/Dockerfile.ubuntu2204 b/connect-content-init/Dockerfile.ubuntu2204 index ac1e8be53..b6533d0b3 100644 --- a/connect-content-init/Dockerfile.ubuntu2204 +++ b/connect-content-init/Dockerfile.ubuntu2204 @@ -1,5 +1,5 @@ # Tag from https://hub.docker.com/_/ubuntu/ -FROM ubuntu:22.04 +FROM ubuntu:22.04 as build LABEL maintainer="RStudio Connect " # Install required tools: @@ -23,3 +23,10 @@ RUN mkdir -p /rsc-staging && \ COPY --chmod=755 entrypoint.sh /usr/local/bin/entrypoint.sh CMD ["/usr/local/bin/entrypoint.sh"] + +FROM build as test + +ARG RSC_VERSION=2024.02.0 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/connect-content-init/Justfile b/connect-content-init/Justfile deleted file mode 100644 index 3c59f2c92..000000000 --- a/connect-content-init/Justfile +++ /dev/null @@ -1,50 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "connect-content-init" -IMAGE_OS := "ubuntu2204" - -RSC_VERSION := "2024.02.0" -RSC_TAG_SAFE_VERSION := replace(RSC_VERSION, "+", "-") -RSC_LICENSE := "" - -R_VERSION := "3.6.2" -R_VERSION_ALT := "4.1.0" - -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" - -DEFAULT_TAG := IMAGE_PREFIX + PRODUCT + ":" + IMAGE_OS + "-" + RSC_TAG_SAFE_VERSION - -# Build Connect content init image - just build ubuntu2204 2022.10.0 rstudio/rstudio-connect-content-init:ubuntu2204-2022.10.0 -build OS=IMAGE_OS VERSION=RSC_VERSION +TAGS=DEFAULT_TAG: - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - tag_array=() - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSC_VERSION="{{ VERSION }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Connect content init image - just test rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test TAG=DEFAULT_TAG VERSION=RSC_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSC_VERSION="{{ VERSION }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Connect content init image interactively - just test-i rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test-i TAG=DEFAULT_TAG VERSION=RSC_VERSION: - just test {{ TAG }} {{ VERSION }} bash diff --git a/connect-content-init/docker-compose.test.yml b/connect-content-init/docker-compose.test.yml deleted file mode 100644 index 0ee6998b4..000000000 --- a/connect-content-init/docker-compose.test.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSC_VERSION - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/connect-content-init/test/run_tests.sh b/connect-content-init/test/run_tests.sh index ff787bcad..139739f34 100755 --- a/connect-content-init/test/run_tests.sh +++ b/connect-content-init/test/run_tests.sh @@ -2,19 +2,13 @@ # install goss -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.8} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/connect/Dockerfile.ubuntu2204 b/connect/Dockerfile.ubuntu2204 index ead6a71d9..7b0a2c69e 100644 --- a/connect/Dockerfile.ubuntu2204 +++ b/connect/Dockerfile.ubuntu2204 @@ -1,10 +1,11 @@ +# syntax=docker/dockerfile:1-labs ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM product-base-pro as build LABEL maintainer="RStudio Docker " COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh @@ -44,3 +45,16 @@ VOLUME ["/data"] ENTRYPOINT ["tini", "--"] CMD ["/usr/local/bin/startup.sh"] + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.12.1 +ARG PYTHON_VERSION_ALT=3.11.7 +ARG RSC_VERSION=2024.02.0 +ARG OS=ubuntu2204 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + --security=insecure \ + /tmp/test/run_tests.sh diff --git a/connect/Justfile b/connect/Justfile deleted file mode 100644 index 29503409b..000000000 --- a/connect/Justfile +++ /dev/null @@ -1,90 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "connect" -IMAGE_OS := "ubuntu2204" - -RSC_VERSION := "2024.02.0" -RSC_LICENSE := "" -RSC_LICENSE_SERVER := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -PERSIST_LICENSE := "false" -PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") - -DEFAULT_TAG := IMAGE_PREFIX + PRODUCT + ":" + IMAGE_OS + "-" + RSC_VERSION - -# Build Connect image - just build ubuntu2204 2022.10.0 rstudio/rstudio-connect:ubuntu2204-2022.10.0 -build OS=IMAGE_OS VERSION=RSC_VERSION +TAGS=DEFAULT_TAG: - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - tag_array=() - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSC_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Connect image - just test rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test TAG=DEFAULT_TAG VERSION=RSC_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSC_VERSION="{{ VERSION }}" \ - RSC_LICENSE="{{ RSC_LICENSE }}" \ - RSC_LICENSE_SERVER="{{ RSC_LICENSE_SERVER }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Connect image interactively - just test-i rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test-i TAG=DEFAULT_TAG VERSION=RSC_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Connect - just RSC_LICENSE="" run rstudio/rstudio-connect:ubuntu2204-2022.10.0 -run TAG=DEFAULT_TAG CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSC_LICENSE }}" ] && [ -z "{{ RSC_LICENSE_SERVER }}" ]; then - echo "Please set RSC_LICENSE or RSC_LICENSE_SERVER before running." - exit 1 - fi - - volume_opts=() - if [ {{ PERSIST_LICENSE }} = "true" ]; then - if [ {{RSC_LICENSE}} ]; then - echo "Volumes will be configured to persist license state data for an activation key." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes key {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - elif [ {{RSC_LICENSE_SERVER}} ]; then - echo "Volumes will be configured to persist license state data for a floating license server." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes float {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - fi - fi - - docker run -it --privileged \ - ${volume_opts[@]} \ - -p 3939:3939 \ - -e RSC_LICENSE="{{ RSC_LICENSE }}" \ - -e RSC_LICENSE_SERVER="{{ RSC_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/connect/docker-compose.test.yml b/connect/docker-compose.test.yml deleted file mode 100644 index 7ac851a5e..000000000 --- a/connect/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - privileged: true - environment: - # uses .env by default - - RSC_VERSION - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - RSC_LICENSE - - RSC_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/connect/test/run_tests.sh b/connect/test/run_tests.sh index 250f2161f..eb91cfaf9 100755 --- a/connect/test/run_tests.sh +++ b/connect/test/run_tests.sh @@ -10,19 +10,13 @@ sleep 15 echo '--> Startup complete' -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/content/docker-bake.hcl b/content/docker-bake.hcl new file mode 100644 index 000000000..3205ca24d --- /dev/null +++ b/content/docker-bake.hcl @@ -0,0 +1,82 @@ +variable CONTENT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu1804", os_alt = "bionic", r = "3.1.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.2.5", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.3.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.3", py = "3.10.4", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu2204", os_alt = "jammy", r = "3.6.3", py = "3.8.16", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.0.5", py = "3.9.16", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.1.3", py = "3.10.11", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.2.2", py = "3.11.3", drivers = "2024.03.0", quarto = "1.3.340"}, + ] + } +} + +group "default" { + targets = ["base", "pro"] +} + +target "base" { + name = "content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + + tags = [ + "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", + "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", + "docker.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", + "docker.io.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", + ] + output = [ + "type=image", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "base" + + matrix = CONTENT_BUILD_MATRIX + args = { + R_VERSION = "${builds.r}" + PYTHON_VERSION = "${builds.py}" + QUARTO_VERSION = "${builds.quarto}" + } +} + +target "pro" { + name = "content-pro-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + + tags = [ + "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", + "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", + "docker.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", + "docker.io.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", + ] + output = [ + "type=image", + ] + + contexts = { + content-base = "target:content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + } + + dockerfile = "Dockerfile.${builds.os}" + context = "pro" + + matrix = CONTENT_BUILD_MATRIX + args = { + R_VERSION = "${builds.r}" + DRIVERS_VERSION = "${builds.drivers}" + } +} diff --git a/content/matrix.json b/content/matrix.json deleted file mode 100644 index 163b8ddd0..000000000 --- a/content/matrix.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - {"r": "3.1.3", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.2.5", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.3.3", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.4.4", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.4.4", "py": "3.7.10", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.5.3", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.5.3", "py": "3.7.10", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "3.8.8", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.7.10", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.8.8", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.9.2", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.1.0", "py": "3.8.8", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.1.0", "py": "3.9.2", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.1.3", "py": "3.10.4", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "3.8.16", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"}, - {"r": "4.0.5", "py": "3.9.16", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"}, - {"r": "4.1.3", "py": "3.10.11", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"}, - {"r": "4.2.2", "py": "3.11.3", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"} -] diff --git a/content/pro/Dockerfile.ubuntu1804 b/content/pro/Dockerfile.ubuntu1804 index e3be3d7fd..1efd4901c 100644 --- a/content/pro/Dockerfile.ubuntu1804 +++ b/content/pro/Dockerfile.ubuntu1804 @@ -1,7 +1,4 @@ -ARG PYTHON_VERSION -ARG R_VERSION -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/content-base:r${R_VERSION}-py${PYTHON_VERSION}-ubuntu1804 +FROM content-base LABEL maintainer="RStudio Docker " diff --git a/content/pro/Dockerfile.ubuntu2204 b/content/pro/Dockerfile.ubuntu2204 index 4a8066b54..1efd4901c 100644 --- a/content/pro/Dockerfile.ubuntu2204 +++ b/content/pro/Dockerfile.ubuntu2204 @@ -1,7 +1,4 @@ -ARG PYTHON_VERSION -ARG R_VERSION -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/content-base:r${R_VERSION}-py${PYTHON_VERSION}-ubuntu2204 +FROM content-base LABEL maintainer="RStudio Docker " diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 000000000..c2ce91ccc --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,677 @@ +### Variable definitions ### +variable CONNECT_VERSION { + default = "2024.02.0" +} + +variable PACKAGE_MANAGER_VERSION { + default = "2023.12.0-13" +} + +variable WORKBENCH_VERSION { + default = "2023.12.1+402.pro1" +} + +variable DRIVERS_VERSION { + default = "2024.03.0" +} + +variable DEFAULT_QUARTO_VERSION { + default = "1.4.553" +} + +function workbench_version_clean { + params = [] + result = split("+", WORKBENCH_VERSION)[0] +} + +function get_os_alt_name { + params = [os] + result = os == "ubuntu2204" ? "jammy" : os +} + +function get_drivers_version { + params = [os] + result = os == "centos7" ? "${DRIVERS_VERSION}-1" : DRIVERS_VERSION +} + +function get_centos_tags { + params = [os, product, product_version] + result = [ + "ghcr.io/rstudio/${product}:${os}", + "ghcr.io/rstudio/${product}:${os}-${product_version}", + "docker.io/rstudio/${product}:${os}", + "docker.io/rstudio/${product}:${os}-${product_version}", + ] +} + +function get_ubuntu_tags { + params = [os, product, product_version] + result = [ + "ghcr.io/rstudio/${product}:${os}", + "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}", + "ghcr.io/rstudio/${product}:${os}-${product_version}", + "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}-${product_version}", + "docker.io/rstudio/${product}:${os}", + "docker.io/rstudio/${product}:${get_os_alt_name(os)}", + "docker.io/rstudio/${product}:${os}-${product_version}", + "docker.io/rstudio/${product}:${get_os_alt_name(os)}-${product_version}", + ] +} + +# FIXME: There's an obnoxious amount of hardcoding here due to restrictions with what bake can actually interpret from HCL. +function get_tags { + params = [os, product, product_version] + result = os == "ubuntu2204" ? get_ubuntu_tags(os, product, product_version) : get_centos_tags(os, product, product_version) +} + +### Build matrices ### + +variable BASE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.0", r_alternate = "3.6.2", py_primary = "3.9.5", py_alternate = "3.8.10"}, + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.2.0", r_alternate = "3.6.2", py_primary = "3.9.5", py_alternate = "3.8.10"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, + ] + } +} + +variable PRO_BUILD_MATRIX { + default = BASE_BUILD_MATRIX +} + +variable PACKAGE_MANAGER_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, + ] + } +} + +variable CONNECT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17"}, + ] + } +} + +variable CONNECT_CONTENT_INIT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204"}, + ] + } +} + +variable R_SESSION_COMPLETE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + ] + } +} + +variable WORKBENCH_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + ] + } +} + +variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2004", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.7", py_alternate = "3.10.13"}, + ] + } +} + +variable WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + ] + } +} + +### Group definitions ### +group "default" { + targets = [ + "product-base", + "product-base-pro", + "connect", + "connect-content-init", + "package-manager", + "r-session-complete", + "workbench", + ] +} + +group "build" { + targets = [ + "product-base", + "product-base-pro", + "connect", + "connect-content-init", + "package-manager", + "r-session-complete", + "workbench", + ] +} + +group "build-test" { + targets = [ + "product-base", + "product-base-pro", + "connect", + "connect-content-init", + "package-manager", + "r-session-complete", + "workbench", + "test-product-base", + "test-product-base-pro", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + "test-connect-content-init", + "test-package-manager", + "test-r-session-complete", + "test-workbench", + "test-workbench-for-google-cloud-workstations", + "test-workbench-for-microsoft-azure-ml", + ] +} + +group "test" { + targets = [ + "test-product-base", + "test-product-base-pro", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + "test-connect-content-init", + "test-package-manager", + "test-r-session-complete", + "test-workbench", + "test-workbench-for-google-cloud-workstations", + "test-workbench-for-microsoft-azure-ml", + ] +} + +group "base-images" { + targets = [ + "product-base", + "test-product-base", + "product-base-pro", + "test-product-base-pro", + ] +} + +group "package-manager-images" { + targets = [ + "package-manager", + "test-package-manager", + ] +} + +group "connect-images" { + targets = [ + "connect", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + ] +} + +group "connect-content-init-images" { + targets = [ + "connect-content-init", + "test-connect-content-init", + ] +} + +group "r-session-complete-images" { + targets = [ + "r-session-complete", + "test-r-session-complete", + ] +} + +group "workbench-images" { + targets = [ + "workbench", + "test-workbench", + ] +} + +group "wgcw-images" { + targets = [ + "workbench-for-google-cloud-workstations", + "test-workbench-for-google-cloud-workstations", + ] +} + +group "waml-images" { + targets = [ + "build-workbench-for-microsoft-azure-ml", + "scan-workbench-for-microsoft-azure-ml", + "test-workbench-for-microsoft-azure-ml", + "workbench-for-microsoft-azure-ml", + ] +} + +### Base Image targets ### +target "base" { + labels = { + "maintainer" = "Posit Docker " + } + output = ["type=image"] +} + +target "product-base" { + inherits = ["base"] + target = "build" + + name = "product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/base" + + matrix = BASE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +target "test-product-base" { + inherits = ["product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "test-product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = BASE_BUILD_MATRIX +} + +target "product-base-pro" { + inherits = ["base"] + target = "build" + + name = "product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/pro" + contexts = { + product-base = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + DRIVERS_VERSION = get_drivers_version(builds.os) + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +target "test-product-base-pro" { + inherits = ["product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "test-product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX +} + +### Package Manager targets ### +target "package-manager" { + inherits = ["base"] + target = "build" + + name = "package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-package-manager", PACKAGE_MANAGER_VERSION) + + dockerfile = "Dockerfile.${builds.os}" + context = "package-manager" + contexts = { + product-base = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PACKAGE_MANAGER_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSPM_VERSION = PACKAGE_MANAGER_VERSION + } +} + +target "test-package-manager" { + inherits = ["package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}"] + target = "test" + + name = "test-package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" + } + + matrix = PACKAGE_MANAGER_BUILD_MATRIX +} + +### Connect targets ### +target "connect" { + inherits = ["base"] + target = "build" + + name = "connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect", CONNECT_VERSION) + + # We output Connect to OCI so it can be pulled in for testing later on. + output = [ + "type=image", + "type=oci,tar=false,dest=./.out/connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "connect" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = CONNECT_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSC_VERSION = CONNECT_VERSION + } +} + +# FIXME: This target requires a privileged environment which bake cannot provide +target "test-connect" { + inherits = ["connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}"] + target = "test" + + name = "test-connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + } + + matrix = CONNECT_BUILD_MATRIX +} + +target "connect-content-init" { + inherits = ["base"] + target = "build" + + name = "connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect-content-init", CONNECT_VERSION) + + dockerfile = "Dockerfile.${builds.os}" + context = "connect-content-init" + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX + + args = { + RSC_VERSION = CONNECT_VERSION + } +} + +target "test-connect-content-init" { + inherits = ["connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}"] + target = "test" + + name = "test-connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + } + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX +} + +### Workbench targets ### +target "r-session-complete" { + inherits = ["base"] + target = "build" + + name = "r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = get_tags(builds.os, "r-session-complete", workbench_version_clean()) + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_VERSION + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "test-r-session-complete" { + inherits = ["r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] + target = "test" + + name = "test-r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX +} + +target "workbench" { + inherits = ["base"] + + name = "workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench", workbench_version_clean()) + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + RSW_VERSION = WORKBENCH_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "test-workbench" { + inherits = ["workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] + + name = "test-workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX +} + +### Workbench for Google Cloud Workstations targets ### +target "workbench-for-google-cloud-workstations" { + inherits = ["base"] + + name = "workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = [ + "us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench:${workbench_version_clean()}", + "us-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "us-docker.pkg.dev/posit-images/cloud-workstations/workbench:${workbench_version_clean()}", + "europe-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "europe-docker.pkg.dev/posit-images/cloud-workstations/workbench:${workbench_version_clean()}", + "asia-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "asia-docker.pkg.dev/posit-images/cloud-workstations/workbench:${workbench_version_clean()}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench-for-google-cloud-workstations" + + matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + QUARTO_VERSION = DEFAULT_QUARTO_VERSION + DRIVERS_VERSION = get_drivers_version(builds.os) + RSW_VERSION = WORKBENCH_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/focal/amd64" + } +} + +target "test-workbench-for-google-cloud-workstations" { + inherits = ["workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] + + name = "test-workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + } + + matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX +} + +### Workbench for Microsoft Azure ML targets ### + +target "build-workbench-for-microsoft-azure-ml" { + inherits = ["base"] + target = "build" + + name = "build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench-for-microsoft-azure-ml" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "scan-workbench-for-microsoft-azure-ml" { + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] + target = "clamav" + + name = "scan-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + + contexts = { + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX +} + +target "test-workbench-for-microsoft-azure-ml" { + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] + target = "test" + + name = "test-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + output = [] + no-cache = true + + contexts = { + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX +} + +target "workbench-for-microsoft-azure-ml" { + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] + target = "final" + + name = "workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench-for-microsoft-azure-ml", workbench_version_clean()) + + contexts = { + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + clamav = "target:scan-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX +} \ No newline at end of file diff --git a/docker-bake.preview.hcl b/docker-bake.preview.hcl new file mode 100644 index 000000000..be8440e97 --- /dev/null +++ b/docker-bake.preview.hcl @@ -0,0 +1,644 @@ +### Variable definitions ### +variable BRANCH { + default = "dev" +} + +variable CONNECT_DAILY_VERSION { + default = null +} + +variable PACKAGE_MANAGER_DAILY_VERSION { + default = null +} + +variable WORKBENCH_DAILY_VERSION { + default = null +} + +variable WORKBENCH_PREVIEW_VERSION { + default = null +} + +variable DRIVERS_VERSION { + default = "2024.03.0" +} + +variable DEFAULT_QUARTO_VERSION { + default = "1.4.553" +} + +variable RSW_PREVIEW_URL_BASE { + default = "https://s3.amazonaws.com/rstudio-ide-build/server/" +} + +function get_rsw_download_url { + params = [os] + result = os == "centos7" ? "${RSW_PREVIEW_URL_BASE}centos7/x86_64" : "${RSW_PREVIEW_URL_BASE}jammy/amd64" +} + +function tag_safe_version { + params = [version] + result = replace(version, "+", "-") +} + +function clean_version { + params = [version] + result = regex_replace(version, "[+|-].*", "") +} + +function get_tag_prefix { + params = [] + result = BRANCH != "main" ? "${BRANCH}-" : "" +} + +function get_drivers_version { + params = [os] + result = os == "centos7" ? "${DRIVERS_VERSION}-1" : DRIVERS_VERSION +} + +function get_os_alt_name { + params = [os] + result = os == "ubuntu2204" ? "jammy" : os +} + +function get_centos_tags { + params = [os, product, product_version, build_type] + result = [ + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + ] +} + +function get_ubuntu_tags { + params = [os, product, product_version, build_type] + result = [ + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${build_type}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${clean_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${build_type}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${clean_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${tag_safe_version(product_version)}", + ] +} + +# FIXME: There's an obnoxious amount of hardcoding in tag-related functions due to restrictions with what bake can actually interpret from HCL. +function get_tags { + params = [os, product, product_version, build_type] + result = os == "ubuntu2204" ? get_ubuntu_tags(os, product, product_version, build_type) : get_centos_tags(os, product, product_version, build_type) +} + +### Build matrices ### +variable BASE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable PRO_BUILD_MATRIX { + default = BASE_BUILD_MATRIX +} + +variable PACKAGE_MANAGER_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable CONNECT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable CONNECT_CONTENT_INIT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204"}, + ] + } +} + +variable R_SESSION_COMPLETE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable WORKBENCH_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +### Group definitions ### +group "default" { + targets = [ + "product-base-dev", + "product-base-pro-dev", + "connect-daily", + "connect-content-init-daily", + "package-manager-daily", + "r-session-complete-preview", + "r-session-complete-daily", + "workbench-preview", + "workbench-daily", + ] +} + +group "build" { + targets = [ + "product-base-dev", + "product-base-pro-dev", + "connect-daily", + "connect-content-init-daily", + "package-manager-daily", + "r-session-complete-preview", + "r-session-complete-daily", + "workbench-preview", + "workbench-daily", + ] +} + +group "build-test" { + targets = [ + "product-base-dev", + "product-base-pro-dev", + "connect-daily", + "connect-content-init-daily", + "package-manager-daily", + "r-session-complete-preview", + "r-session-complete-daily", + "workbench-preview", + "workbench-daily", + "test-product-base-dev", + "test-product-base-pro-dev", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + "test-connect-content-init-daily", + "test-package-manager-daily", + "test-r-session-complete-daily", + "test-r-session-complete-preview", + "test-workbench-daily", + "test-workbench-preview", + ] +} + +group "test" { + targets = [ + "test-product-base-dev", + "test-product-base-pro-dev", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + "test-connect-content-init-daily", + "test-package-manager-daily", + "test-r-session-complete-daily", + "test-r-session-complete-preview", + "test-workbench-daily", + "test-workbench-preview", + ] +} + +group "base-dev-images" { + targets = [ + "product-base-dev", + "test-product-base-dev", + "product-base-pro-dev", + "test-product-base-pro-dev", + ] +} + +group "package-manager-daily-images" { + targets = [ + "package-manager-daily", + "test-package-manager-daily", + ] +} + +group "connect-daily-images" { + targets = [ + "connect-daily", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + ] +} + +group "connect-content-init-daily-images" { + targets = [ + "connect-content-init-daily", + "test-connect-content-init-daily", + ] +} + +group "r-session-complete-images" { + targets = [ + "r-session-complete-daily", + "r-session-complete-preview", + "test-r-session-complete-daily", + "test-r-session-complete-preview", + ] +} + +group "workbench-images" { + targets = [ + "workbench-daily", + "workbench-preview", + "test-workbench-daily", + "test-workbench-preview", + ] +} + +### Dev Base Image targets ### +target "base" { + labels = { + "maintainer" = "Posit Docker " + } + output = ["type=image"] +} + +target "product-base-dev" { + inherits = ["base"] + target = "build" + + name = "product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/base" + + matrix = BASE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +target "test-product-base-dev" { + inherits = ["product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "test-product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = BASE_BUILD_MATRIX +} + +target "product-base-pro-dev" { + inherits = ["base"] + target = "build" + + name = "product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-pro-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-pro-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/pro" + contexts = { + product-base = "target:product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + DRIVERS_VERSION = get_drivers_version(builds.os) + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +target "test-product-base-pro-dev" { + inherits = ["product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "test-product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX +} + +### Package Manager targets ### +target "package-manager-daily" { + inherits = ["base"] + target = "build" + + name = "package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-package-manager-preview", PACKAGE_MANAGER_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "package-manager" + contexts = { + product-base = "target:product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PACKAGE_MANAGER_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSPM_VERSION = PACKAGE_MANAGER_DAILY_VERSION + } +} + +target "test-package-manager-daily" { + inherits = ["package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}"] + target = "test" + + name = "test-package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" + } + + matrix = PACKAGE_MANAGER_BUILD_MATRIX +} + +### Connect targets ### +target "connect-daily" { + inherits = ["base"] + target = "build" + + name = "connect-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect-preview", CONNECT_DAILY_VERSION, "daily") + + # We output Connect to OCI so it can be pulled in for testing later on. + output = [ + "type=image", + "type=oci,tar=false,dest=./.out/connect-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "connect" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = CONNECT_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSC_VERSION = CONNECT_DAILY_VERSION + } +} + +# FIXME: This target requires a privileged environment which bake cannot provide +target "test-connect-daily" { + inherits = ["connect-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}"] + target = "test" + + name = "test-connect-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:connect-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + } + + matrix = CONNECT_BUILD_MATRIX +} + +target "connect-content-init-daily" { + inherits = ["base"] + target = "build" + + name = "connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect-content-init-preview", CONNECT_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "connect-content-init" + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX + + args = { + RSC_VERSION = CONNECT_DAILY_VERSION + } +} + +target "test-connect-content-init-daily" { + inherits = ["connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}"] + target = "test" + + name = "test-connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + } + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX +} + +### Workbench targets ### +target "r-session-complete-daily" { + inherits = ["base"] + target = "build" + + name = "r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "r-session-complete-preview", WORKBENCH_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_DAILY_VERSION + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) + } +} + +target "test-r-session-complete-daily" { + inherits = ["r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}"] + target = "test" + + name = "test-r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX +} + +target "r-session-complete-preview" { + inherits = ["base"] + target = "build" + + name = "r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = get_tags(builds.os, "r-session-complete-preview", WORKBENCH_PREVIEW_VERSION, "preview") + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_PREVIEW_VERSION + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) + } +} + +target "test-r-session-complete-preview" { + inherits = ["r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}"] + target = "test" + + name = "test-r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX +} + +target "workbench-daily" { + inherits = ["base"] + + name = "workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench-preview", WORKBENCH_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + RSW_VERSION = WORKBENCH_DAILY_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) + } +} + +target "test-workbench-daily" { + inherits = ["workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}"] + + name = "test-workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX +} + +target "workbench-preview" { + inherits = ["base"] + + name = "workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench-preview", WORKBENCH_PREVIEW_VERSION, "preview") + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + RSW_VERSION = WORKBENCH_PREVIEW_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) + } +} + +target "test-workbench-preview" { + inherits = ["workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}"] + + name = "test-workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = [] + output = [] + no-cache = true + + contexts = { + build = "target:workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX +} diff --git a/docker-compose.yml b/docker-compose.yml index 149b95df2..88ef5b309 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: dockerfile: "Dockerfile.${IMAGE_OS:-ubuntu2204}" args: RSW_VERSION: 2023.12.1+402.pro1 - image: rstudio/rstudio-workbench:2023.12.0 + image: rstudio/rstudio-workbench:2023.12.1 environment: RSW_LICENSE: ${RSW_LICENSE} LICENSE_SERVER: ${RSW_LICENSE_SERVER} diff --git a/hadolint.yaml b/hadolint.yaml index bf071e06c..6548cc9c6 100644 --- a/hadolint.yaml +++ b/hadolint.yaml @@ -1,5 +1,7 @@ failure-threshold: warning ignored: + # Suppresses warning for not explicitly tagging FROM images. Docker buildx bake is handling FROM image injection. + - DL3006 # Suppresses warning for version pinning on apt installs. We currently do not practice this approach. - DL3008 # Suppresses warning for version pinning on yum installs. We currently do not practice this approach. diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204 index 06c052be2..d3bc84741 100644 --- a/package-manager/Dockerfile.ubuntu2204 +++ b/package-manager/Dockerfile.ubuntu2204 @@ -4,8 +4,8 @@ ARG PYTHON_VERSION=3.12.1 ARG PYTHON_VERSION_ALT=3.11.7 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} -LABEL maintainer="RStudio Docker " +FROM product-base as build +LABEL maintainer="Posit Docker " ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 @@ -50,3 +50,15 @@ RUN license-manager initialize --userspace || true ENTRYPOINT ["tini", "--"] CMD ["/usr/local/bin/startup.sh"] + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.12.1 +ARG PYTHON_VERSION_ALT=3.11.7 +ARG RSPM_VERSION=2023.12.0-13 +ARG OS=ubuntu2204 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/package-manager/Justfile b/package-manager/Justfile deleted file mode 100644 index cb03a2f40..000000000 --- a/package-manager/Justfile +++ /dev/null @@ -1,98 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "package-manager" -IMAGE_OS := "ubuntu2204" - -RSPM_VERSION := "2023.12.0-13" -RSPM_LICENSE := "" -RSPM_LICENSE_SERVER := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.12.1" -PYTHON_VERSION_ALT := "3.11.7" - -PERSIST_LICENSE := "false" -PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSPM_VERSION}})" - -# Build Package Manager image - just build ubuntu2204 2022.07.2-11 rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 -build OS=IMAGE_OS VERSION=RSPM_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSPM_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Package Manager image - just test rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 2022.07.2-11 -test TAG=`just _make-default-tag` VERSION=RSPM_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSPM_VERSION="{{ VERSION }}" \ - RSPM_LICENSE="{{ RSPM_LICENSE }}" \ - RSPM_LICENSE_SERVER="{{ RSPM_LICENSE_SERVER }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Package Manager image interactively - just test-i rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 2022.07.2-11 -test-i TAG=`just _make-default-tag` VERSION=RSPM_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Package Manager - just RSPM_LICENSE="" run rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSPM_LICENSE }}" ] && [ -z "{{ RSPM_LICENSE_SERVER }}" ]; then - echo "Please set RSPM_LICENSE or RSPM_LICENSE_SERVER before running." - exit 1 - fi - - volume_opts=() - if [ {{ PERSIST_LICENSE }} = "true" ]; then - if [ {{RSPM_LICENSE}} ]; then - echo "Volumes will be configured to persist license state data for an activation key." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes key {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - elif [ {{RSPM_LICENSE_SERVER}} ]; then - echo "Volumes will be configured to persist license state data for a floating license server." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes float {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - fi - fi - - docker run -it \ - ${volume_opts[@]} \ - -p 4242:4242 \ - -e RSPM_LICENSE="{{ RSPM_LICENSE }}" \ - -e RSPM_LICENSE_SERVER="{{ RSPM_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/package-manager/docker-compose.test.yml b/package-manager/docker-compose.test.yml deleted file mode 100644 index d1279b981..000000000 --- a/package-manager/docker-compose.test.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSPM_VERSION - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - RSPM_LICENSE - - RSPM_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/package-manager/test/run_tests.sh b/package-manager/test/run_tests.sh index 98227a949..f106fe857 100755 --- a/package-manager/test/run_tests.sh +++ b/package-manager/test/run_tests.sh @@ -1,5 +1,6 @@ #!/bin/bash +# start connect trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/startup.log; exit $err' ERR # start package manager @@ -8,21 +9,13 @@ tini -- /usr/local/bin/startup.sh >/tmp/startup.log 2>&1 & echo '--> Waiting for startup' sleep 15 -echo '--> Startup (hopefully) complete' - -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/product/base/Dockerfile.centos7 b/product/base/Dockerfile.centos7 index 273410824..3107153c5 100644 --- a/product/base/Dockerfile.centos7 +++ b/product/base/Dockerfile.centos7 @@ -1,4 +1,4 @@ -FROM centos:7 +FROM centos:7 as build LABEL maintainer="Posit Docker " ### ARG declarations ### @@ -98,3 +98,16 @@ LABEL posit.r.version="${R_VERSION}" \ posit.quarto.version="${QUARTO_VERSION}" ENTRYPOINT ["/tini", "--"] + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG TINI_VERSION=0.19.0 +ARG QUARTO_VERSION=1.3.340 +ARG OS=centos7 + +COPY test/ /tmp/test +RUN /tmp/test/run_tests.sh diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index f326b3e31..d67c2dd37 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:22.04 as build LABEL maintainer="Posit Docker " ### ARG declarations ### @@ -146,3 +146,17 @@ LABEL posit.r.version="${R_VERSION}" \ posit.quarto.version="${QUARTO_VERSION}" ENTRYPOINT ["/tini", "--"] + +FROM build as test + +ARG DEBIAN_FRONTEND=noninteractive +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG TINI_VERSION=0.19.0 +ARG QUARTO_VERSION=1.3.340 +ARG OS=ubuntu2204 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/product/base/Justfile b/product/base/Justfile deleted file mode 100755 index 00c603788..000000000 --- a/product/base/Justfile +++ /dev/null @@ -1,69 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -PRODUCT := "product-base" -IMAGE_OS := "ubuntu2204" - -IMAGE_REGISTRY := "rstudio" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -TINI_VERSION := "0.19.0" -QUARTO_VERSION := "1.3.340" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_REGISTRY}}/{{PRODUCT}}:{{OS}}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}}" - -# Build base image - just build ubuntu2204 rstudio/product-base:ubuntu2204 -build OS=IMAGE_OS *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg TINI_VERSION="{{ TINI_VERSION }}" \ - --build-arg QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - --file=./Dockerfile.$(just -f ../../Justfile _parse-os {{OS}}) . - -# Test base image - just test rstudio/product-base:ubuntu2204 -test TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - TINI_VERSION="{{ TINI_VERSION }}" \ - OS="{{ IMAGE_OS }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test base image interactively - just test-i rstudio/product-base:ubuntu2204 -test-i TAG=`just _make-default-tag`: - just test {{ TAG }} bash diff --git a/product/base/docker-compose.test.yml b/product/base/docker-compose.test.yml deleted file mode 100644 index bbd42874b..000000000 --- a/product/base/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - privileged: true - environment: - # uses .env by default - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - TINI_VERSION - - QUARTO_VERSION - - OS - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/product/base/test/run_tests.sh b/product/base/test/run_tests.sh index 1f355886e..c96df6568 100755 --- a/product/base/test/run_tests.sh +++ b/product/base/test/run_tests.sh @@ -5,19 +5,13 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start echo '--> Startup complete' -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/product/pro/Dockerfile.centos7 b/product/pro/Dockerfile.centos7 index 0214e2ab9..98ba632b7 100644 --- a/product/pro/Dockerfile.centos7 +++ b/product/pro/Dockerfile.centos7 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.14 ARG PYTHON_VERSION_ALT=3.8.15 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM product-base as build LABEL maintainer="Posit Docker " ARG R_VERSION=4.2.0 @@ -25,3 +25,17 @@ RUN yum update -y \ LABEL rstudio.pro-drivers.version="${DRIVERS_VERSION}" ENTRYPOINT ["/tini", "--"] + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG TINI_VERSION=0.19.0 +ARG QUARTO_VERSION=1.3.340 +ARG DRIVERS_VERSION=2024.03.0-1 +ARG OS=centos7 + +COPY test/ /tmp/test +RUN /tmp/test/run_tests.sh diff --git a/product/pro/Dockerfile.ubuntu2204 b/product/pro/Dockerfile.ubuntu2204 index 3914cffbb..b78a1aeae 100644 --- a/product/pro/Dockerfile.ubuntu2204 +++ b/product/pro/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM product-base as build LABEL maintainer="Posit Docker " ARG DEBIAN_FRONTEND=noninteractive @@ -27,3 +27,18 @@ RUN apt-get update \ LABEL rstudio.pro-drivers.version="${DRIVERS_VERSION}" ENTRYPOINT ["/tini", "--"] + +FROM build as test + +ARG DEBIAN_FRONTEND=noninteractive +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG TINI_VERSION=0.19.0 +ARG QUARTO_VERSION=1.3.340 +ARG DRIVERS_VERSION=2024.03.0 +ARG OS=ubuntu2204 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/product/pro/Justfile b/product/pro/Justfile deleted file mode 100644 index d07474f01..000000000 --- a/product/pro/Justfile +++ /dev/null @@ -1,79 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -PRODUCT := "product-base-pro" -IMAGE_OS := "ubuntu2204" - -IMAGE_REGISTRY := "rstudio" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -TINI_VERSION := "0.19.0" -QUARTO_VERSION := "1.3.340" - -DRIVERS_VERSION := "2024.03.0" -DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_REGISTRY}}/{{PRODUCT}}:{{OS}}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}}" - -# Build base pro image - just build ubuntu2204 rstudio/product-base-pro:ubuntu2204 -build OS=IMAGE_OS *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - if [[ "{{ OS }}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - tag_array=() - for TAG in $raw_tag_array - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg DRIVERS_VERSION="${_DRIVERS_VERSION}" \ - --file=./Dockerfile.$(just -f ../../Justfile _parse-os {{OS}}) . - -# Test base image - just test rstudio/product-base-pro:ubuntu2204 -test TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - - IMAGE_NAME="{{ TAG }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - TINI_VERSION="{{ TINI_VERSION }}" \ - DRIVERS_VERSION="{{ DRIVERS_VERSION }}" \ - OS="{{ IMAGE_OS }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test base image interactively - just test-i rstudio/product-base-pro:ubuntu2204 -test-i TAG=`just _make-default-tag`: - just test {{ TAG }} bash diff --git a/product/pro/docker-compose.test.yml b/product/pro/docker-compose.test.yml deleted file mode 100644 index e34b86fa5..000000000 --- a/product/pro/docker-compose.test.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - privileged: true - environment: - # uses .env by default - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - TINI_VERSION - - QUARTO_VERSION - - DRIVERS_VERSION - - DRIVERS_VERSION_RHEL - - OS - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/product/pro/test/goss.yaml b/product/pro/test/goss.yaml index 53ea2a34a..8670095d9 100644 --- a/product/pro/test/goss.yaml +++ b/product/pro/test/goss.yaml @@ -22,7 +22,12 @@ package: rstudio-drivers: installed: true versions: - - {{ trimSuffix "-1" .Env.DRIVERS_VERSION }} # RHEL driver doesn't print the "-1" suffix in the package name + {{if .Env.OS | regexMatch "ubuntu.*"}} + - {{ .Env.DRIVERS_VERSION }} # RHEL driver doesn't print the "-1" suffix in the package name + {{end}} + {{if .Env.OS | regexMatch "centos.*"}} + - {{ .Env.DRIVERS_VERSION }}.el + {{end}} file: /opt/R/{{.Env.R_VERSION}}/bin/R: diff --git a/product/pro/test/run_tests.sh b/product/pro/test/run_tests.sh index 1f355886e..c96df6568 100755 --- a/product/pro/test/run_tests.sh +++ b/product/pro/test/run_tests.sh @@ -5,19 +5,13 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start echo '--> Startup complete' -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/r-session-complete/Dockerfile.centos7 b/r-session-complete/Dockerfile.centos7 index ae766c0ba..4d5c08bf3 100644 --- a/r-session-complete/Dockerfile.centos7 +++ b/r-session-complete/Dockerfile.centos7 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.14 ARG PYTHON_VERSION_ALT=3.8.15 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM product-base-pro as build LABEL maintainer="RStudio Docker "` ARG R_VERSION=4.2.3 @@ -57,3 +57,15 @@ ENV PATH="/opt/python/${PYTHON_VERSION}/bin:${PATH}" COPY vscode.extensions.conf /etc/rstudio/vscode.extensions.conf EXPOSE 8788/tcp + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG RSW_VERSION=2023.12.1+402.pro1 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/r-session-complete/Dockerfile.ubuntu2204 b/r-session-complete/Dockerfile.ubuntu2204 index 1f3018b24..7e3f02060 100644 --- a/r-session-complete/Dockerfile.ubuntu2204 +++ b/r-session-complete/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM product-base-pro as build LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive @@ -66,3 +66,15 @@ ENV PATH="/opt/python/${PYTHON_VERSION}/bin:${PATH}" COPY vscode.extensions.conf /etc/rstudio/vscode.extensions.conf EXPOSE 8788/tcp + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG RSW_VERSION=2023.12.1+402.pro1 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/r-session-complete/Justfile b/r-session-complete/Justfile deleted file mode 100755 index da24d8304..000000000 --- a/r-session-complete/Justfile +++ /dev/null @@ -1,87 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "" -PRODUCT := "r-session-complete" -IMAGE_OS := "ubuntu2204" - -RSW_VERSION := "2023.12.1+402.pro1" -RSW_LICENSE := "" - -DRIVERS_VERSION := "2024.03.0" -DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.14" -PYTHON_VERSION_ALT := "3.8.15" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" - -# Build r-session-complete image - just build ubuntu2204 2022.07.2+576.pro12 rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -build OS=IMAGE_OS VERSION=RSW_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - if [[ "{{ OS }}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ - --build-arg DRIVERS_VERSION="${_DRIVERS_VERSION}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test r-session-complete image - just test rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test r-session-complete image interactively - just test-i rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run r-session-complete - just run rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSW_LICENSE }}" ]; then - echo "Please set RSW_LICENSE to a valid RStudio Workbench license before running." - exit 1 - fi - docker run -it --privileged \ - -p 8788:8788 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/r-session-complete/test/goss.yaml b/r-session-complete/test/goss.yaml index 94637931e..09760e7c0 100644 --- a/r-session-complete/test/goss.yaml +++ b/r-session-complete/test/goss.yaml @@ -22,6 +22,7 @@ file: command: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/{{.Env.PYTHON_VERSION}}/bin/jupyter nbconvert --to notebook --stdin --stdout": title: jupyter_works + timeout: 60000 exit-status: 0 # Ensure correct R version @@ -49,4 +50,5 @@ command: "jupyter --version": title: jupyter_in_path_var + timeout: 60000 exit-status: 0 diff --git a/r-session-complete/test/run_tests.sh b/r-session-complete/test/run_tests.sh index 16568e9c0..0c71dcad3 100755 --- a/r-session-complete/test/run_tests.sh +++ b/r-session-complete/test/run_tests.sh @@ -1,19 +1,12 @@ #!/bin/bash -# install goss - -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/share/buildkitd.toml b/share/buildkitd.toml new file mode 100644 index 000000000..ac746f6f2 --- /dev/null +++ b/share/buildkitd.toml @@ -0,0 +1 @@ +insecure-entitlements = ["security.insecure"] \ No newline at end of file diff --git a/share/local_buildkitd.toml b/share/local_buildkitd.toml new file mode 100644 index 000000000..a775e7ff7 --- /dev/null +++ b/share/local_buildkitd.toml @@ -0,0 +1,4 @@ +insecure-entitlements = ["security.insecure"] + +[worker.oci] + max-parallelism = 4 \ No newline at end of file diff --git a/tools/bake_test_command_extract.py b/tools/bake_test_command_extract.py new file mode 100644 index 000000000..6bf67669b --- /dev/null +++ b/tools/bake_test_command_extract.py @@ -0,0 +1,65 @@ +import argparse +import json +import subprocess +from pathlib import Path + +PROJECT_DIR = Path(__file__).resolve().parents[1] + + +parser = argparse.ArgumentParser( + description="Extract a test command from a bake plan" +) +parser.add_argument("image_name") +parser.add_argument("operating_system") +parser.add_argument("--builder", default="posit-builder") +parser.add_argument("--file", default="docker-bake.hcl") + + +def get_bake_plan(bake_file="docker-bake.hcl"): + cmd = ["docker", "buildx", "bake", "-f", str(PROJECT_DIR / bake_file), "--print"] + p = subprocess.run(cmd, capture_output=True) + if p.returncode != 0: + print(f"Failed to get bake plan: {p.stderr}") + exit(1) + return json.loads(p.stdout.decode("utf-8")) + + +def get_targets(plan, image_name, operating_system): + targets = {} + for target_name, target_spec in plan["target"].items(): + if target_name.startswith(f"{image_name}-{operating_system}"): + targets[target_name] = target_spec + return targets + + +def build_test_command(target_name, target_spec, builder): + cmd = [ + "docker", + "buildx", + "--builder", + builder, + "build", + "--allow", + "security.insecure", + "--target", + "test", + ] + cmd.extend(["--build-context", f"build=oci-layout://{PROJECT_DIR / '.out' / target_name}"]) + for name, value in target_spec["args"].items(): + cmd.extend(["--build-arg", f'{name}="{value}"']) + cmd.extend(["--file", str(PROJECT_DIR / target_spec["context"] / target_spec["dockerfile"])]) + cmd.append(str(PROJECT_DIR / target_spec["context"])) + return cmd + + +def main(): + args = parser.parse_args() + plan = get_bake_plan(args.file) + targets = get_targets(plan, args.image_name, args.operating_system) + for target_name, target_spec in targets.items(): + cmd = build_test_command(target_name, target_spec, args.builder) + print(" ".join(cmd)) + + +if __name__ == "__main__": + main() diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 index b612a6a97..e7211a2fc 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 @@ -1,4 +1,4 @@ -FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:public-image-current +FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:public-image-current as build ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive @@ -147,8 +147,7 @@ RUN /opt/python/"${PYTHON_VERSION_JUPYTER}"/bin/python -m venv /opt/python/jupyt && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge -RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ - && chmod +x /usr/local/bin/wait-for-it.sh +ADD --chmod=755 https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh /usr/local/bin/wait-for-it.sh RUN mkdir -p /var/lib/rstudio-server/monitor/log \ && chown -R rstudio-server:rstudio-server /var/lib/rstudio-server/monitor \ @@ -179,3 +178,22 @@ RUN apt-get remove -yq dpkg-sig \ EXPOSE 80/tcp EXPOSE 5559/tcp + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.11.7 +ARG PYTHON_VERSION_ALT=3.10.13 +ARG PYTHON_VERSION_JUPYTER=3.10.13 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG QUARTO_VERSION=1.3.450 +ARG DRIVERS_VERSION=2023.05.0 +ARG RSW_VERSION=2023.12.1+402.pro1 +ARG DOCKER_TEST_STAGE=true + +COPY deps/py_packages.txt /tmp/py_packages.txt +COPY deps/r_packages.txt /tmp/r_packages.txt +RUN apt-get update && apt-get install -yq net-tools +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile deleted file mode 100644 index ce026a86f..000000000 --- a/workbench-for-google-cloud-workstations/Justfile +++ /dev/null @@ -1,116 +0,0 @@ -set dotenv-load -set positional-arguments - -BUILDX_PATH := "" - -PRODUCT := "workbench" -IMAGE_OS := "ubuntu2004" - -RSW_LICENSE := "" -RSW_LICENSE_SERVER := "" - -_make-default-tag: - echo "${IMAGE_REGISTRY_NAME}:$(just -f ../Justfile _get-tag-safe-version "${RSW_VERSION}")" - -get-version: - echo "${RSW_VERSION}" - -get-build-args: - #!/usr/bin/env bash - printf "RSW_VERSION=${RSW_VERSION} - R_VERSION=${R_VERSION} - R_VERSION_ALT=${R_VERSION_ALT} - PYTHON_VERSION=${PYTHON_VERSION} - PYTHON_VERSION_ALT=${PYTHON_VERSION_ALT} - PYTHON_VERSION_JUPYTER=${PYTHON_VERSION_JUPYTER} - QUARTO_VERSION=${QUARTO_VERSION} - DRIVERS_VERSION=${DRIVERS_VERSION} - SRC_IMAGE_NAME=${SRC_IMAGE_NAME} - RSW_DOWNLOAD_URL=${RSW_DOWNLOAD_URL}" - -get-build-tags: - #!/usr/bin/env bash - set -eu - regions=("us-central1" "asia" "europe" "us") - tag_array=() - for region in "${regions[@]}"; do - tag_array+=("$region-docker.pkg.dev/posit-images/cloud-workstations/workbench:${RSW_TAG_VERSION}") - tag_array+=("$region-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest") - done - IFS="," - echo "${tag_array[*]}" - -# Build Workbench image - just build 2022.07.2+576.pro12 rstudio/rstudio-workbench-gcw:2022.07.2 -build *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - CACHE_PATH=${GITHUB_WORKSPACE:-/tmp} - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=${CACHE_PATH}/.buildx-cache --cache-to=type=local,dest=${CACHE_PATH}/.buildx-cache-new,mode=max" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag)) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in ${raw_tag_array[@]}; - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="${RSW_VERSION}" \ - --build-arg R_VERSION="${R_VERSION}" \ - --build-arg R_VERSION_ALT="${R_VERSION_ALT}" \ - --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ - --build-arg PYTHON_VERSION_ALT="${PYTHON_VERSION_ALT}" \ - --build-arg PYTHON_VERSION_JUPYTER="${PYTHON_VERSION_JUPYTER}" \ - --build-arg QUARTO_VERSION="${QUARTO_VERSION}" \ - --build-arg DRIVERS_VERSION="${DRIVERS_VERSION}" \ - --build-arg RSW_DOWNLOAD_URL="${RSW_DOWNLOAD_URL}" \ - --file=./Dockerfile.ubuntu2004 . - - echo ${raw_tag_array[@]} - -# Test Workbench image - just test rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - RSW_VERSION_CLEAN=$(sed "s/daily-/daily+/" <<<"${RSW_VERSION}") - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="${RSW_VERSION_CLEAN}" \ - RSW_LICENSE="{{ RSW_LICENSE }}" \ - RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - DRIVERS_VERSION="${DRIVERS_VERSION}" \ - R_VERSION="${R_VERSION}" \ - R_VERSION_ALT="${R_VERSION_ALT}" \ - PYTHON_VERSION="${PYTHON_VERSION}" \ - PYTHON_VERSION_ALT="${PYTHON_VERSION_ALT}" \ - PYTHON_VERSION_JUPYTER="${PYTHON_VERSION_JUPYTER}" \ - QUARTO_VERSION="${QUARTO_VERSION}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag`: - just test {{ TAG }} bash - -# Run Workbench - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu1804-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euo pipefail - if [ -z "{{ RSW_LICENSE }}" ] && [ -z "{{ RSW_LICENSE_SERVER }}" ]; then - echo "Please set RSW_LICENSE or RSW_LICENSE_SERVER before running." - exit 1 - fi - - docker run -it --privileged \ - ${volume_opts[@]} \ - -p 8787:80 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - -e RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench-for-google-cloud-workstations/docker-compose.test.yml b/workbench-for-google-cloud-workstations/docker-compose.test.yml deleted file mode 100644 index 8de098c6d..000000000 --- a/workbench-for-google-cloud-workstations/docker-compose.test.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSW_VERSION - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - PYTHON_VERSION_JUPYTER - - QUARTO_VERSION - - DRIVERS_VERSION - - RSW_LICENSE - - RSW_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" - - "./deps/py_packages.txt:/tmp/py_packages.txt" - - "./deps/r_packages.txt:/tmp/r_packages.txt" diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index 674665ef4..9209c995c 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -35,19 +35,14 @@ port: tcp:80: listening: true ip: - - 0.0.0.0 - skip: false - tcp:5559: - listening: true - ip: - - 127.0.0.1 + - 0.0.0.0 skip: false process: rserver: running: true skip: false - rstudio-launche: + rserver-launche: running: true skip: false @@ -88,14 +83,14 @@ file: exists: true /etc/rstudio/jupyter.conf: exists: true - contains: + contents: - --LabApp.token="" - --no-browser - --allow-root - --ip=0.0.0.0 /etc/rstudio/rserver.conf: exists: true - contains: + contents: - www-port=80 - launcher-sessions-callback-address=http://0.0.0.0:80 - auth-proxy=1 @@ -103,21 +98,21 @@ file: - auth-proxy-user-header=x-custom-user-name /etc/rstudio/nginx.site.conf: exists: true - contains: + contents: - proxy_set_header X-CUSTOM-USER-NAME user/google; - proxy_set_header Host $http_x_forwarded_host; /etc/rstudio/vscode.extensions.conf: exists: true - contains: + contents: - quarto.quarto - GoogleCloudTools.cloudcode /tmp/startup.log: exists: true - contains: + contents: - "!Error reading /etc/rstudio/rserver.conf:" /etc/pam.d/common-session: exists: true - contains: + contents: - "/^session required pam_mkhomedir.so skel=/etc/skel umask=0022$/" /etc/sssd/sssd.conf: exists: true @@ -135,7 +130,7 @@ file: group: root mode: "0755" filetype: file - contains: [ + contents: [ "useradd -m user" ] /etc/workstation-startup.d/110_config-jupyter.sh: @@ -144,7 +139,7 @@ file: group: root mode: "0755" filetype: file - contains: [ + contents: [ # Checks that we're setting the Jupyter shell to /bin/bash "echo \"c.ServerApp.terminado_settings = {'shell_command': ['/bin/bash']}\" > /home/user/.jupyter/jupyter_notebook_config.py" ] @@ -154,7 +149,7 @@ file: group: root mode: "0755" filetype: file - contains: [ + contents: [ "/usr/bin/supervisord -c /etc/supervisor/supervisord.conf" ] @@ -182,21 +177,25 @@ command: ] Test Jupyter works: exec: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout" + timeout: 60000 exit-status: 0 Check primary Python is version {{.Env.PYTHON_VERSION}}: exec: /opt/python/{{.Env.PYTHON_VERSION}}/bin/python --version + timeout: 60000 exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION}}" ] Check alternate Python is version {{.Env.PYTHON_VERSION_ALT}}: exec: /opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version + timeout: 60000 exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION_ALT}}" ] Check Jupyter venv uses Python {{.Env.PYTHON_VERSION_JUPYTER}}: exec: /opt/python/jupyter/bin/python --version + timeout: 60000 exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION_JUPYTER}}" @@ -213,9 +212,11 @@ command: {{- range $py_package_list }} Check Python {{ $python_version }} has "{{.}}" installed: exec: /opt/python/{{$python_version}}/bin/pip show {{.}} + timeout: 60000 exit-status: 0 Check Python {{ $python_version_alt }} has "{{.}}" installed: exec: /opt/python/{{$python_version_alt}}/bin/pip show {{.}} + timeout: 60000 exit-status: 0 {{end}} {{ $r_version := .Env.R_VERSION }} diff --git a/workbench-for-google-cloud-workstations/test/run_tests.sh b/workbench-for-google-cloud-workstations/test/run_tests.sh index 02c2cd04f..72f4c1c6e 100755 --- a/workbench-for-google-cloud-workstations/test/run_tests.sh +++ b/workbench-for-google-cloud-workstations/test/run_tests.sh @@ -9,25 +9,22 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & +sleep 15 + +netstat -tulpn echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:80 -t $RSW_TIMEOUT wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.22} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} - -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5} # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 index 4a8156945..b964c5f37 100644 --- a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 +++ b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} AS workbench +FROM product-base-pro AS build LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive @@ -115,8 +115,7 @@ RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install -r /tmp/py_packages && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \ && rm /tmp/py_packages.txt -RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ - && chmod +x /usr/local/bin/wait-for-it.sh +ADD --chmod=755 https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh /usr/local/bin/wait-for-it.sh RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash \ && az extension add -n ml -y @@ -130,9 +129,22 @@ EXPOSE 5559/tcp ENTRYPOINT [] CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] +FROM build AS test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG PYTHON_VERSION_JUPYTER=3.8.17 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG RSW_VERSION=2023.12.1+402.pro1 +ARG DOCKER_TEST_STAGE=true + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh -FROM workbench AS clamav +FROM build AS clamav # Set up ClamAV RUN apt-get update \ @@ -169,7 +181,7 @@ RUN clamscan \ # with ClamAV installed. Avoid adding changes in this stage unless they are related # to the ClamAV stage. Since the ClamAV stage may be cached, you can't guarantee # another full scan if you change anything beyond this point. -FROM workbench AS final +FROM build AS final # Copy ClamAV scan logs so the end user can see them. COPY --from=clamav /var/log/clamav/clamscan.log /var/log/clamav/clamupdate.log / diff --git a/workbench-for-microsoft-azure-ml/Justfile b/workbench-for-microsoft-azure-ml/Justfile deleted file mode 100644 index cd9a2fa08..000000000 --- a/workbench-for-microsoft-azure-ml/Justfile +++ /dev/null @@ -1,78 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "workbench" -IMAGE_OS := "ubuntu2204" - -RSW_VERSION := "2023.12.1+402.pro1" -RSW_LICENSE := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" - -# Build Workbench for Azure ML image - just build ubuntu2204 2022.07.2+576.pro12 rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 -build OS=IMAGE_OS VERSION=RSW_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Workbench for Azure ML image - just test rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Workbench for Azure ML image interactively - just test-i rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Workbench for Azure ML - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSW_LICENSE }}" ]; then - echo "Please set RSW_LICENSE to a valid RStudio Workbench license before running." - exit 1 - fi - docker run -it --privileged \ - -p 8787:8787 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench-for-microsoft-azure-ml/docker-compose.test.yml b/workbench-for-microsoft-azure-ml/docker-compose.test.yml deleted file mode 100644 index 53a1d86d0..000000000 --- a/workbench-for-microsoft-azure-ml/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSW_VERSION - - R_VERSION - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - PYTHON_VERSION_JUPYTER - - R_VERSION_ALT - - RSW_LICENSE - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" - - "./test/goss_vars.yaml:/tmp/goss_vars.yaml" diff --git a/workbench-for-microsoft-azure-ml/test/goss.yaml b/workbench-for-microsoft-azure-ml/test/goss.yaml index 05269e7d9..2c8ba3fea 100644 --- a/workbench-for-microsoft-azure-ml/test/goss.yaml +++ b/workbench-for-microsoft-azure-ml/test/goss.yaml @@ -101,6 +101,7 @@ command: ] "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": title: jupyter_works + timeout: 60000 exit-status: 0 # Ensure correct python version "/opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version": diff --git a/workbench-for-microsoft-azure-ml/test/goss_vars.yaml b/workbench-for-microsoft-azure-ml/test/goss_vars.yaml index 2669ba18b..65f2d26c5 100644 --- a/workbench-for-microsoft-azure-ml/test/goss_vars.yaml +++ b/workbench-for-microsoft-azure-ml/test/goss_vars.yaml @@ -19,7 +19,6 @@ syspkgs: - libglib2.0-0 - libpq5 - libsm6 - - libssl-dev - openssl - libssl-dev - libuser diff --git a/workbench-for-microsoft-azure-ml/test/run_tests.sh b/workbench-for-microsoft-azure-ml/test/run_tests.sh index 9dc299cb2..894196850 100755 --- a/workbench-for-microsoft-azure-ml/test/run_tests.sh +++ b/workbench-for-microsoft-azure-ml/test/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash set -xe -RSW_TIMEOUT=${RSW_TIMEOUT:-15} +RSW_TIMEOUT=${RSW_TIMEOUT:-60} touch /tmp/startup.log trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/startup.log; exit $err' ERR @@ -9,24 +9,21 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & +sleep 15 echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT +wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} - -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VARS_FILE=${GOSS_VARS_FILE:-/tmp/test/goss_vars.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5} # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/workbench/Dockerfile.ubuntu2204 b/workbench/Dockerfile.ubuntu2204 index 64edfed12..7511c72ba 100644 --- a/workbench/Dockerfile.ubuntu2204 +++ b/workbench/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM product-base-pro as build LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive @@ -108,3 +108,19 @@ EXPOSE 5559/tcp ENTRYPOINT [] CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG PYTHON_VERSION_JUPYTER=3.8.17 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG RSW_VERSION=2023.12.1+402.pro1 +ARG DOCKER_TEST_STAGE=true + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh + + diff --git a/workbench/Justfile b/workbench/Justfile deleted file mode 100644 index 785e0ad75..000000000 --- a/workbench/Justfile +++ /dev/null @@ -1,100 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "workbench" -IMAGE_OS := "ubuntu2204" - -RSW_VERSION := "2023.12.1+402.pro1" -RSW_RELEASE_TYPE := "release" -RSW_LICENSE := "" -RSW_LICENSE_SERVER := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -PERSIST_LICENSE := "false" -PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") - -_make-default-tag OS=IMAGE_OS VERSION=RSW_VERSION: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{VERSION}})" - -# Build Workbench image - just build ubuntu2204 2022.07.2+576.pro12 rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 -build OS=IMAGE_OS VERSION=RSW_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}} {{VERSION}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg RSW_DOWNLOAD_URL="$(just -f ../ci.Justfile _get-rsw-download-url {{RSW_RELEASE_TYPE}} {{OS}})" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Workbench image - just test rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ - RSW_LICENSE="{{ RSW_LICENSE }}" \ - RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Workbench - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSW_LICENSE }}" ] && [ -z "{{ RSW_LICENSE_SERVER }}" ]; then - echo "Please set RSW_LICENSE or RSW_LICENSE_SERVER before running." - exit 1 - fi - - volume_opts=() - if [ {{ PERSIST_LICENSE }} = "true" ]; then - if [ {{RSW_LICENSE}} ]; then - echo "Volumes will be configured to persist license state data for an activation key." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes key {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - elif [ {{RSW_LICENSE_SERVER}} ]; then - echo "Volumes will be configured to persist license state data for a floating license server." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes float {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - fi - fi - - docker run -it --privileged \ - ${volume_opts[@]} \ - -p 8787:8787 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - -e RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench/docker-compose.test.yml b/workbench/docker-compose.test.yml deleted file mode 100644 index db54a879d..000000000 --- a/workbench/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSW_VERSION - - R_VERSION - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - PYTHON_VERSION_JUPYTER - - R_VERSION_ALT - - RSW_LICENSE - - RSW_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/workbench/test/goss.yaml b/workbench/test/goss.yaml index 0a9a78c70..21379f9e7 100644 --- a/workbench/test/goss.yaml +++ b/workbench/test/goss.yaml @@ -78,6 +78,7 @@ file: exists: true {{ end }} /var/lib/rstudio-launcher/Local/jobs/buildkitsandbox: + skip: {{ .Env.DOCKER_TEST_STAGE }} # This file is present during the build stage, but not after exists: false /etc/rstudio/vscode.conf: exists: true @@ -112,6 +113,7 @@ command: ] "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": title: jupyter_works + timeout: 60000 exit-status: 0 # Ensure correct python version "/opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version": diff --git a/workbench/test/run_tests.sh b/workbench/test/run_tests.sh index 9dc299cb2..00bba3847 100755 --- a/workbench/test/run_tests.sh +++ b/workbench/test/run_tests.sh @@ -9,24 +9,22 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & +sleep 15 echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT +wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} +ps aux | grep rs -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5} # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT