From ca7dbb4701fed5e7b00c0d3a0e8cdc68823d327c Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 12 Mar 2024 10:46:31 -0400 Subject: [PATCH 01/37] Initial commit of docker buildx bake experiment --- .github/workflows/build-bake.yaml | 23 +++ docker-bake.hcl | 327 ++++++++++++++++++++++++++++++ 2 files changed, 350 insertions(+) create mode 100644 .github/workflows/build-bake.yaml create mode 100644 docker-bake.hcl diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml new file mode 100644 index 000000000..61922b661 --- /dev/null +++ b/.github/workflows/build-bake.yaml @@ -0,0 +1,23 @@ +on: + push: + branches: + - main + - dev + pull_request: + +name: Build Docker Images with Bake +jobs: + build: + runs-on: ubuntu-latest-4x + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/bake-action@v4 + with: + push: false + diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 000000000..32a2ea535 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,327 @@ +variable BASE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.0", r_alternate = "3.6.2", py_primary = "3.9.5", py_alternate = "3.8.10"}, + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.2.0", r_alternate = "3.6.2", py_primary = "3.9.5", py_alternate = "3.8.10"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, + ] + } +} + +variable PRO_BUILD_MATRIX { + default = BASE_BUILD_MATRIX +} + +variable PACKAGE_MANAGER_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, + ] + } +} + +variable CONNECT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17"}, + ] + } +} + +variable CONNECT_CONTENT_INIT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204"}, + ] + } +} + +variable R_SESSION_COMPLETE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + ] + } +} + +variable WORKBENCH_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + ] + } +} + +variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2004", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, + ] + } +} + +variable WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + ] + } +} + +group "default" { + targets = [ + "product-base", + "product-base-pro", + "connect", + "connect-content-init", + "package-manager", + "r-session-complete", + "workbench", + "workbench-for-google-cloud-workstations", + "workbench-for-microsoft-azure-ml", + ] +} + +target "base" { + labels = { + "maintainer" = "RStudio Docker " + } +} + +target "product-base" { + inherits = ["base"] + name = "product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/base" + + matrix = BASE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + } +} + +target "product-base-pro" { + inherits = ["base"] + + name = "product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/pro" + contexts = { + baseapp = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + DEBIAN_FRONTEND = "noninteractive" + DRIVERS_VERSION = builds.os == "centos7" ? "2023.12.1-1" : "2023.12.1" + } +} + + +target "package-manager" { + inherits = ["base"] + + name = "package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "package-manager" + contexts = { + baseapp = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PACKAGE_MANAGER_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSPM_VERSION = "2023.12.0-13" + } +} + +target "connect" { + inherits = ["base"] + + name = "connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "connect" + contexts = { + baseapp = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = CONNECT_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSC_VERSION = "2024.02.0" + } +} + +target "connect-content-init" { + inherits = ["base"] + + name = "connect-content-init-${builds.os}" + tags = [ + "ghcr.io/rstudio/product-connect-content-init:${builds.os}", + "docker.io/rstudio/product-connect-content-init:${builds.os}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "connect-content-init" + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX + + args = { + DEBIAN_FRONTEND = "noninteractive" + RSC_VERSION = "2024.02.0" + } +} + +target "r-session-complete" { + inherits = ["base"] + + name = "r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-r-session-complete:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-r-session-complete:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + contexts = { + baseapp = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + DEBIAN_FRONTEND = "noninteractive" + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = "2023.12.1+402.pro1" + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "workbench" { + inherits = ["base"] + + name = "workbench-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-workbench:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-workbench:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench" + contexts = { + baseapp = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX + args = { + DEBIAN_FRONTEND = "noninteractive" + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + RSW_VERSION = "2023.12.1+402.pro1" + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "workbench-for-google-cloud-workstations" { + inherits = ["base"] + + name = "workbench-for-google-cloud-workstation-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-workbench-for-google-cloud-workstation:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-workbench-for-google-cloud-workstation:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench-for-google-cloud-workstations" + + matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX + args = { + DEBIAN_FRONTEND = "noninteractive" + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + QUARTO_VERSION = "1.4.550" + DRIVERS_VERSION = "2023.05.0" + RSW_VERSION = "2023.12.1+402.pro1" + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/focal/amd64" + } +} + +target "workbench-for-microsoft-azure-ml" { + inherits = ["base"] + + name = "workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-workbench-for-microsoft-azure-ml:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-workbench-for-microsoft-azure-ml:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench-for-microsoft-azure-ml" + contexts = { + baseapp = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX + args = { + DEBIAN_FRONTEND = "noninteractive" + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = "2023.12.1+402.pro1" + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + } +} \ No newline at end of file From b8c32a2de6cafb4790ee212fa3f425c9169f6439 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 12 Mar 2024 11:26:53 -0400 Subject: [PATCH 02/37] Separate builds for azure/google specialized images --- .github/workflows/build-bake.yaml | 7 +++++++ docker-bake.hcl | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 61922b661..a3c82e91a 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -9,6 +9,13 @@ name: Build Docker Images with Bake jobs: build: runs-on: ubuntu-latest-4x + strategy: + matrix: + targets: [ + "default", + "workbench-for-google-cloud-workstations", + "workbench-for-microsoft-azure-ml", + ] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/docker-bake.hcl b/docker-bake.hcl index 32a2ea535..d2fea3b63 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -81,8 +81,6 @@ group "default" { "package-manager", "r-session-complete", "workbench", - "workbench-for-google-cloud-workstations", - "workbench-for-microsoft-azure-ml", ] } From 57938e5136892f91ba847db3f1e9cb7f2132d4ed Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 12 Mar 2024 11:27:43 -0400 Subject: [PATCH 03/37] Specify targets in bake --- .github/workflows/build-bake.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index a3c82e91a..daf510bfa 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -26,5 +26,6 @@ jobs: - name: Build and push uses: docker/bake-action@v4 with: + targets: "${{ matrix.targets }}" push: false From 77cd9e8629c32cafd8d689834e7cb517d5a88725 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 2 Apr 2024 15:21:20 -0700 Subject: [PATCH 04/37] Implement test stages on base --- docker-bake.hcl | 127 +++++++++++++++++++++++------ product/base/Dockerfile.centos7 | 16 +++- product/base/Dockerfile.ubuntu2204 | 16 +++- product/base/test/run_tests.sh | 12 +-- 4 files changed, 135 insertions(+), 36 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index d2fea3b63..f04ad4d7d 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,3 +1,24 @@ +variable CONNECT_VERSION { + default = "2024.02.0" +} + +variable PACKAGE_MANAGER_VERSION { + default = "2023.12.0-13" +} + +variable WORKBENCH_VERSION { + default = "2023.12.1+402.pro1" +} + +variable DRIVERS_VERSION { + default = "2024.03.0" +} + +function get_drivers_version { + params = [os] + result = os == "centos7" ? "${DRIVERS_VERSION}-1" : DRIVERS_VERSION +} + variable BASE_BUILD_MATRIX { default = { builds = [ @@ -74,24 +95,33 @@ variable WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX { group "default" { targets = [ - "product-base", - "product-base-pro", - "connect", + "product-base", + "product-base-pro", + "connect", "connect-content-init", - "package-manager", + "package-manager", "r-session-complete", - "workbench", + "workbench", + ] +} + +group "build-test-base" { + targets = [ + "product-base", + "test-product-base", ] } target "base" { labels = { - "maintainer" = "RStudio Docker " + "maintainer" = "Posit Docker " } } target "product-base" { inherits = ["base"] + target = "build" + name = "product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [ "ghcr.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", @@ -107,11 +137,34 @@ target "product-base" { R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" } } +target "test-product-base" { + inherits = ["product-base"] + target = "test" + + name = "test-product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + + dockerfile = "Dockerfile.${builds.os}" + context = "product/base" + + matrix = BASE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + target "product-base-pro" { inherits = ["base"] + target = "build" name = "product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [ @@ -131,11 +184,40 @@ target "product-base-pro" { R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate - DEBIAN_FRONTEND = "noninteractive" - DRIVERS_VERSION = builds.os == "centos7" ? "2023.12.1-1" : "2023.12.1" + DRIVERS_VERSION = get_drivers_version(builds.os) + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" } } +target "test-product-base-pro" { + inherits = ["product-base-pro"] + target = "test" + + name = "product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/pro" + contexts = { + baseapp = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + DRIVERS_VERSION = get_drivers_version(builds.os) + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + target "package-manager" { inherits = ["base"] @@ -154,11 +236,11 @@ target "package-manager" { matrix = PACKAGE_MANAGER_BUILD_MATRIX args = { - R_VERSION = builds.r_primary - R_VERSION_ALT = builds.r_alternate - PYTHON_VERSION = builds.py_primary - PYTHON_VERSION_ALT = builds.py_alternate - RSPM_VERSION = "2023.12.0-13" + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSPM_VERSION = PACKAGE_MANAGER_VERSION } } @@ -183,7 +265,7 @@ target "connect" { R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate - RSC_VERSION = "2024.02.0" + RSC_VERSION = CONNECT_VERSION } } @@ -202,8 +284,7 @@ target "connect-content-init" { matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX args = { - DEBIAN_FRONTEND = "noninteractive" - RSC_VERSION = "2024.02.0" + RSC_VERSION = CONNECT_VERSION } } @@ -224,13 +305,12 @@ target "r-session-complete" { matrix = R_SESSION_COMPLETE_BUILD_MATRIX args = { - DEBIAN_FRONTEND = "noninteractive" R_VERSION = builds.r_primary R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate JUPYTERLAB_VERSION = "3.6.5" - RSW_VERSION = "2023.12.1+402.pro1" + RSW_VERSION = WORKBENCH_VERSION RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" } @@ -253,13 +333,12 @@ target "workbench" { matrix = WORKBENCH_BUILD_MATRIX args = { - DEBIAN_FRONTEND = "noninteractive" R_VERSION = builds.r_primary R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate PYTHON_VERSION_JUPYTER = builds.py_alternate - RSW_VERSION = "2023.12.1+402.pro1" + RSW_VERSION = WORKBENCH_VERSION RSW_NAME = "rstudio-workbench" RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" } @@ -279,7 +358,6 @@ target "workbench-for-google-cloud-workstations" { matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX args = { - DEBIAN_FRONTEND = "noninteractive" R_VERSION = builds.r_primary R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary @@ -287,8 +365,8 @@ target "workbench-for-google-cloud-workstations" { PYTHON_VERSION_JUPYTER = builds.py_alternate JUPYTERLAB_VERSION = "3.6.5" QUARTO_VERSION = "1.4.550" - DRIVERS_VERSION = "2023.05.0" - RSW_VERSION = "2023.12.1+402.pro1" + DRIVERS_VERSION = get_drivers_version(builds.os) + RSW_VERSION = WORKBENCH_VERSION RSW_NAME = "rstudio-workbench" RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/focal/amd64" } @@ -311,14 +389,13 @@ target "workbench-for-microsoft-azure-ml" { matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX args = { - DEBIAN_FRONTEND = "noninteractive" R_VERSION = builds.r_primary R_VERSION_ALT = builds.r_alternate PYTHON_VERSION = builds.py_primary PYTHON_VERSION_ALT = builds.py_alternate PYTHON_VERSION_JUPYTER = builds.py_alternate JUPYTERLAB_VERSION = "3.6.5" - RSW_VERSION = "2023.12.1+402.pro1" + RSW_VERSION = WORKBENCH_VERSION RSW_NAME = "rstudio-workbench" RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" } diff --git a/product/base/Dockerfile.centos7 b/product/base/Dockerfile.centos7 index 273410824..e0a8a22f0 100644 --- a/product/base/Dockerfile.centos7 +++ b/product/base/Dockerfile.centos7 @@ -1,4 +1,4 @@ -FROM centos:7 +FROM centos:7 as build LABEL maintainer="Posit Docker " ### ARG declarations ### @@ -98,3 +98,17 @@ LABEL posit.r.version="${R_VERSION}" \ posit.quarto.version="${QUARTO_VERSION}" ENTRYPOINT ["/tini", "--"] + +FROM build as test + +ARG DEBIAN_FRONTEND=noninteractive +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG TINI_VERSION=0.19.0 +ARG QUARTO_VERSION=1.3.340 +ARG OS=centos7 + +COPY test/ /tmp/test +RUN /tmp/test/run_tests.sh diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index f326b3e31..d67c2dd37 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:22.04 as build LABEL maintainer="Posit Docker " ### ARG declarations ### @@ -146,3 +146,17 @@ LABEL posit.r.version="${R_VERSION}" \ posit.quarto.version="${QUARTO_VERSION}" ENTRYPOINT ["/tini", "--"] + +FROM build as test + +ARG DEBIAN_FRONTEND=noninteractive +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG TINI_VERSION=0.19.0 +ARG QUARTO_VERSION=1.3.340 +ARG OS=ubuntu2204 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/product/base/test/run_tests.sh b/product/base/test/run_tests.sh index 1f355886e..c96df6568 100755 --- a/product/base/test/run_tests.sh +++ b/product/base/test/run_tests.sh @@ -5,19 +5,13 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start echo '--> Startup complete' -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT From b16433525eb6abb818ec330afea03d382edb65ac Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 3 Apr 2024 06:25:13 -0700 Subject: [PATCH 05/37] Implement stage tests for base --- docker-bake.hcl | 8 +++++--- product/base/Dockerfile.centos7 | 1 - product/pro/Dockerfile.centos7 | 15 ++++++++++++++- product/pro/Dockerfile.ubuntu2204 | 16 +++++++++++++++- product/pro/test/goss.yaml | 7 ++++++- product/pro/test/run_tests.sh | 12 +++--------- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index f04ad4d7d..c07b36e72 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -105,10 +105,12 @@ group "default" { ] } -group "build-test-base" { +group "base-images" { targets = [ "product-base", "test-product-base", + "product-base-pro", + "test-product-base-pro", ] } @@ -143,7 +145,7 @@ target "product-base" { } target "test-product-base" { - inherits = ["product-base"] + inherits = ["product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] target = "test" name = "test-product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" @@ -191,7 +193,7 @@ target "product-base-pro" { } target "test-product-base-pro" { - inherits = ["product-base-pro"] + inherits = ["product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] target = "test" name = "product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" diff --git a/product/base/Dockerfile.centos7 b/product/base/Dockerfile.centos7 index e0a8a22f0..3107153c5 100644 --- a/product/base/Dockerfile.centos7 +++ b/product/base/Dockerfile.centos7 @@ -101,7 +101,6 @@ ENTRYPOINT ["/tini", "--"] FROM build as test -ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.9.17 diff --git a/product/pro/Dockerfile.centos7 b/product/pro/Dockerfile.centos7 index 0214e2ab9..a0ed998e7 100644 --- a/product/pro/Dockerfile.centos7 +++ b/product/pro/Dockerfile.centos7 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.14 ARG PYTHON_VERSION_ALT=3.8.15 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build LABEL maintainer="Posit Docker " ARG R_VERSION=4.2.0 @@ -25,3 +25,16 @@ RUN yum update -y \ LABEL rstudio.pro-drivers.version="${DRIVERS_VERSION}" ENTRYPOINT ["/tini", "--"] + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG TINI_VERSION=0.19.0 +ARG QUARTO_VERSION=1.3.340 +ARG OS=centos7 + +COPY test/ /tmp/test +RUN /tmp/test/run_tests.sh diff --git a/product/pro/Dockerfile.ubuntu2204 b/product/pro/Dockerfile.ubuntu2204 index 3914cffbb..d9335dd7e 100644 --- a/product/pro/Dockerfile.ubuntu2204 +++ b/product/pro/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build LABEL maintainer="Posit Docker " ARG DEBIAN_FRONTEND=noninteractive @@ -27,3 +27,17 @@ RUN apt-get update \ LABEL rstudio.pro-drivers.version="${DRIVERS_VERSION}" ENTRYPOINT ["/tini", "--"] + +FROM build as test + +ARG DEBIAN_FRONTEND=noninteractive +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG TINI_VERSION=0.19.0 +ARG QUARTO_VERSION=1.3.340 +ARG OS=ubuntu2204 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/product/pro/test/goss.yaml b/product/pro/test/goss.yaml index 53ea2a34a..8670095d9 100644 --- a/product/pro/test/goss.yaml +++ b/product/pro/test/goss.yaml @@ -22,7 +22,12 @@ package: rstudio-drivers: installed: true versions: - - {{ trimSuffix "-1" .Env.DRIVERS_VERSION }} # RHEL driver doesn't print the "-1" suffix in the package name + {{if .Env.OS | regexMatch "ubuntu.*"}} + - {{ .Env.DRIVERS_VERSION }} # RHEL driver doesn't print the "-1" suffix in the package name + {{end}} + {{if .Env.OS | regexMatch "centos.*"}} + - {{ .Env.DRIVERS_VERSION }}.el + {{end}} file: /opt/R/{{.Env.R_VERSION}}/bin/R: diff --git a/product/pro/test/run_tests.sh b/product/pro/test/run_tests.sh index 1f355886e..c96df6568 100755 --- a/product/pro/test/run_tests.sh +++ b/product/pro/test/run_tests.sh @@ -5,19 +5,13 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start echo '--> Startup complete' -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT From bf20f9494fd5aa28615ad7f9726dc1a9a163aa7d Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 3 Apr 2024 06:34:49 -0700 Subject: [PATCH 06/37] Implement stage tests for ppm --- docker-bake.hcl | 38 +++++++++++++++++++++++++-- package-manager/Dockerfile.ubuntu2204 | 14 +++++++++- package-manager/test/run_tests.sh | 15 +++-------- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index c07b36e72..58d0b328a 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -114,11 +114,20 @@ group "base-images" { ] } +group "package-manager-images" { + targets = [ + "package-manager", + "test-package-manager", + ] +} + +### Base Images ### + target "base" { labels = { "maintainer" = "Posit Docker " } -} +} target "product-base" { inherits = ["base"] @@ -220,9 +229,11 @@ target "test-product-base-pro" { } } +### Package Manager ### target "package-manager" { - inherits = ["base"] + inherits = ["product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "build" name = "package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [ @@ -246,6 +257,29 @@ target "package-manager" { } } +target "test-package-manager" { + inherits = ["package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "package-manager" + + matrix = PACKAGE_MANAGER_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSPM_VERSION = PACKAGE_MANAGER_VERSION + } +} + target "connect" { inherits = ["base"] diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204 index 06c052be2..fd5d4efe2 100644 --- a/package-manager/Dockerfile.ubuntu2204 +++ b/package-manager/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.12.1 ARG PYTHON_VERSION_ALT=3.11.7 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build LABEL maintainer="RStudio Docker " ARG PYTHON_VERSION=3.9.17 @@ -50,3 +50,15 @@ RUN license-manager initialize --userspace || true ENTRYPOINT ["tini", "--"] CMD ["/usr/local/bin/startup.sh"] + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.12.1 +ARG PYTHON_VERSION_ALT=3.11.7 +ARG RSPM_VERSION=2023.12.0-13 +ARG OS=ubuntu2204 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/package-manager/test/run_tests.sh b/package-manager/test/run_tests.sh index 98227a949..f106fe857 100755 --- a/package-manager/test/run_tests.sh +++ b/package-manager/test/run_tests.sh @@ -1,5 +1,6 @@ #!/bin/bash +# start connect trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/startup.log; exit $err' ERR # start package manager @@ -8,21 +9,13 @@ tini -- /usr/local/bin/startup.sh >/tmp/startup.log 2>&1 & echo '--> Waiting for startup' sleep 15 -echo '--> Startup (hopefully) complete' - -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT From d02d71d5840868ce6b43ca6cc4c83bd3f9072b27 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 3 Apr 2024 08:40:31 -0700 Subject: [PATCH 07/37] Implement stage tests for connect (borked) --- connect-content-init/Dockerfile.ubuntu2204 | 9 ++- connect-content-init/test/run_tests.sh | 12 +-- connect/Dockerfile.ubuntu2204 | 16 +++- connect/test/run_tests.sh | 12 +-- docker-bake.hcl | 87 +++++++++++++++++----- 5 files changed, 96 insertions(+), 40 deletions(-) diff --git a/connect-content-init/Dockerfile.ubuntu2204 b/connect-content-init/Dockerfile.ubuntu2204 index ac1e8be53..b6533d0b3 100644 --- a/connect-content-init/Dockerfile.ubuntu2204 +++ b/connect-content-init/Dockerfile.ubuntu2204 @@ -1,5 +1,5 @@ # Tag from https://hub.docker.com/_/ubuntu/ -FROM ubuntu:22.04 +FROM ubuntu:22.04 as build LABEL maintainer="RStudio Connect " # Install required tools: @@ -23,3 +23,10 @@ RUN mkdir -p /rsc-staging && \ COPY --chmod=755 entrypoint.sh /usr/local/bin/entrypoint.sh CMD ["/usr/local/bin/entrypoint.sh"] + +FROM build as test + +ARG RSC_VERSION=2024.02.0 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/connect-content-init/test/run_tests.sh b/connect-content-init/test/run_tests.sh index ff787bcad..139739f34 100755 --- a/connect-content-init/test/run_tests.sh +++ b/connect-content-init/test/run_tests.sh @@ -2,19 +2,13 @@ # install goss -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.8} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/connect/Dockerfile.ubuntu2204 b/connect/Dockerfile.ubuntu2204 index ead6a71d9..cc11a80cb 100644 --- a/connect/Dockerfile.ubuntu2204 +++ b/connect/Dockerfile.ubuntu2204 @@ -1,10 +1,11 @@ +# syntax=docker/dockerfile:1-labs ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build LABEL maintainer="RStudio Docker " COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh @@ -44,3 +45,16 @@ VOLUME ["/data"] ENTRYPOINT ["tini", "--"] CMD ["/usr/local/bin/startup.sh"] + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.12.1 +ARG PYTHON_VERSION_ALT=3.11.7 +ARG RSC_VERSION=2024.02.0 +ARG OS=ubuntu2204 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + --security=insecure \ + /tmp/test/run_tests.sh diff --git a/connect/test/run_tests.sh b/connect/test/run_tests.sh index 250f2161f..eb91cfaf9 100755 --- a/connect/test/run_tests.sh +++ b/connect/test/run_tests.sh @@ -10,19 +10,13 @@ sleep 15 echo '--> Startup complete' -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/docker-bake.hcl b/docker-bake.hcl index 58d0b328a..29f3784ce 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -121,7 +121,21 @@ group "package-manager-images" { ] } -### Base Images ### +group "connect-images" { + targets = [ + "connect", + "test-connect", + ] +} + +group "connect-content-init-images" { + targets = [ + "connect-content-init", + "test-connect-content-init", + ] +} + +### Base Image targets ### target "base" { labels = { @@ -205,17 +219,10 @@ target "test-product-base-pro" { inherits = ["product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] target = "test" - name = "product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - tags = [ - "ghcr.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + name = "test-product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" dockerfile = "Dockerfile.${builds.os}" context = "product/pro" - contexts = { - baseapp = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - } matrix = PRO_BUILD_MATRIX args = { @@ -229,7 +236,7 @@ target "test-product-base-pro" { } } -### Package Manager ### +### Package Manager targets ### target "package-manager" { inherits = ["product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] @@ -237,8 +244,8 @@ target "package-manager" { name = "package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [ - "ghcr.io/rstudio/product-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/product-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "ghcr.io/rstudio/rstudio-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/rstudio-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", ] dockerfile = "Dockerfile.${builds.os}" @@ -261,11 +268,7 @@ target "test-package-manager" { inherits = ["package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] target = "test" - name = "package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - tags = [ - "ghcr.io/rstudio/product-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/product-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + name = "test-package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" dockerfile = "Dockerfile.${builds.os}" context = "package-manager" @@ -280,13 +283,16 @@ target "test-package-manager" { } } +### Connect targets ### + target "connect" { - inherits = ["base"] + inherits = ["product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "build" name = "connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [ - "ghcr.io/rstudio/product-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/product-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "ghcr.io/rstudio/rstudio-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/rstudio-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", ] dockerfile = "Dockerfile.${builds.os}" @@ -305,8 +311,29 @@ target "connect" { } } +# FIXME: This target requires a privileged environment which bake cannot provide +target "test-connect" { + inherits = ["connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "test-connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + + dockerfile = "Dockerfile.${builds.os}" + context = "connect" + + matrix = CONNECT_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSC_VERSION = CONNECT_VERSION + } +} + target "connect-content-init" { inherits = ["base"] + target = "build" name = "connect-content-init-${builds.os}" tags = [ @@ -324,6 +351,24 @@ target "connect-content-init" { } } +target "test-connect-content-init" { + inherits = ["connect-content-init-${builds.os}"] + target = "test" + + name = "test-connect-content-init-${builds.os}" + + dockerfile = "Dockerfile.${builds.os}" + context = "connect-content-init" + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX + + args = { + RSC_VERSION = CONNECT_VERSION + } +} + +### Workbench targets ### + target "r-session-complete" { inherits = ["base"] @@ -380,6 +425,8 @@ target "workbench" { } } +### Specialty image targets ### + target "workbench-for-google-cloud-workstations" { inherits = ["base"] From d0946259d58994e44684801cc97a2fc20b6db966 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 3 Apr 2024 10:08:37 -0700 Subject: [PATCH 08/37] Update connect just test to work with test stage --- connect/Justfile | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/connect/Justfile b/connect/Justfile index 29503409b..185940cca 100644 --- a/connect/Justfile +++ b/connect/Justfile @@ -45,22 +45,15 @@ build OS=IMAGE_OS VERSION=RSC_VERSION +TAGS=DEFAULT_TAG: --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . # Test Connect image - just test rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test TAG=DEFAULT_TAG VERSION=RSC_VERSION CMD="": +test OS="ubuntu2204" VERSION=RSC_VERSION: #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSC_VERSION="{{ VERSION }}" \ - RSC_LICENSE="{{ RSC_LICENSE }}" \ - RSC_LICENSE_SERVER="{{ RSC_LICENSE_SERVER }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Connect image interactively - just test-i rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test-i TAG=DEFAULT_TAG VERSION=RSC_VERSION: - just test {{ TAG }} {{ VERSION }} bash + docker buildx --builder="{{ BUILDX_PATH }}" build ${BUILDX_ARGS} --allow security.insecure --target test \ + --build-arg RSC_VERSION="{{ VERSION }}" \ + --build-arg R_VERSION="{{ R_VERSION }}" \ + --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ + --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ + --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ + --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . # Run Connect - just RSC_LICENSE="" run rstudio/rstudio-connect:ubuntu2204-2022.10.0 run TAG=DEFAULT_TAG CMD="": From e552094706e1481220743a6977c1942f932a8ea1 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 3 Apr 2024 10:09:31 -0700 Subject: [PATCH 09/37] Implement test stage on r-session-complete --- docker-bake.hcl | 32 +++++++++++++++++++++++- r-session-complete/Dockerfile.centos7 | 14 ++++++++++- r-session-complete/Dockerfile.ubuntu2204 | 14 ++++++++++- r-session-complete/test/run_tests.sh | 11 +++----- 4 files changed, 60 insertions(+), 11 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 29f3784ce..1c6e16d5d 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -135,6 +135,13 @@ group "connect-content-init-images" { ] } +group "r-session-complete-images" { + targets = [ + "r-session-complete", + "test-r-session-complete", + ] +} + ### Base Image targets ### target "base" { @@ -370,7 +377,8 @@ target "test-connect-content-init" { ### Workbench targets ### target "r-session-complete" { - inherits = ["base"] + inherits = ["product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "build" name = "r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [ @@ -397,6 +405,28 @@ target "r-session-complete" { } } +target "test-r-session-complete" { + inherits = ["r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "test-r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_VERSION + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" + } +} + target "workbench" { inherits = ["base"] diff --git a/r-session-complete/Dockerfile.centos7 b/r-session-complete/Dockerfile.centos7 index ae766c0ba..2165e0d58 100644 --- a/r-session-complete/Dockerfile.centos7 +++ b/r-session-complete/Dockerfile.centos7 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.14 ARG PYTHON_VERSION_ALT=3.8.15 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build LABEL maintainer="RStudio Docker "` ARG R_VERSION=4.2.3 @@ -57,3 +57,15 @@ ENV PATH="/opt/python/${PYTHON_VERSION}/bin:${PATH}" COPY vscode.extensions.conf /etc/rstudio/vscode.extensions.conf EXPOSE 8788/tcp + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG RSW_VERSION=2023.12.1+402.pro1 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/r-session-complete/Dockerfile.ubuntu2204 b/r-session-complete/Dockerfile.ubuntu2204 index 1f3018b24..1a27dd09a 100644 --- a/r-session-complete/Dockerfile.ubuntu2204 +++ b/r-session-complete/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive @@ -66,3 +66,15 @@ ENV PATH="/opt/python/${PYTHON_VERSION}/bin:${PATH}" COPY vscode.extensions.conf /etc/rstudio/vscode.extensions.conf EXPOSE 8788/tcp + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG RSW_VERSION=2023.12.1+402.pro1 + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/r-session-complete/test/run_tests.sh b/r-session-complete/test/run_tests.sh index 16568e9c0..139739f34 100755 --- a/r-session-complete/test/run_tests.sh +++ b/r-session-complete/test/run_tests.sh @@ -2,18 +2,13 @@ # install goss -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT From a4647761fb3b49455cb6819ff1202a5c0bc94b43 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 4 Apr 2024 11:05:28 -0700 Subject: [PATCH 10/37] Fix/exclude goss tests for buildx bake --- .github/workflows/build-bake.yaml | 11 +- connect/Dockerfile.ubuntu2204 | 2 +- docker-bake.hcl | 281 ++++++++++++------ package-manager/Dockerfile.ubuntu2204 | 4 +- product/pro/Dockerfile.centos7 | 3 +- product/pro/Dockerfile.ubuntu2204 | 3 +- r-session-complete/Dockerfile.ubuntu2204 | 2 +- r-session-complete/test/run_tests.sh | 2 - .../Dockerfile.ubuntu2004 | 24 +- .../test/goss.yaml | 27 +- .../test/run_tests.sh | 15 +- .../Dockerfile.ubuntu2204 | 22 +- .../test/goss_vars.yaml | 1 - .../test/run_tests.sh | 17 +- workbench/Dockerfile.ubuntu2204 | 18 +- workbench/test/goss.yaml | 1 + workbench/test/run_tests.sh | 12 +- 17 files changed, 289 insertions(+), 156 deletions(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index daf510bfa..fb7c2a17c 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -9,13 +9,6 @@ name: Build Docker Images with Bake jobs: build: runs-on: ubuntu-latest-4x - strategy: - matrix: - targets: [ - "default", - "workbench-for-google-cloud-workstations", - "workbench-for-microsoft-azure-ml", - ] steps: - name: Checkout uses: actions/checkout@v4 @@ -26,6 +19,6 @@ jobs: - name: Build and push uses: docker/bake-action@v4 with: - targets: "${{ matrix.targets }}" - push: false + targets: "build-test" + push: false diff --git a/connect/Dockerfile.ubuntu2204 b/connect/Dockerfile.ubuntu2204 index cc11a80cb..7b0a2c69e 100644 --- a/connect/Dockerfile.ubuntu2204 +++ b/connect/Dockerfile.ubuntu2204 @@ -5,7 +5,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build +FROM product-base-pro as build LABEL maintainer="RStudio Docker " COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh diff --git a/docker-bake.hcl b/docker-bake.hcl index 1c6e16d5d..a03e7de83 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,3 +1,4 @@ +### Variable definitions ### variable CONNECT_VERSION { default = "2024.02.0" } @@ -19,6 +20,8 @@ function get_drivers_version { result = os == "centos7" ? "${DRIVERS_VERSION}-1" : DRIVERS_VERSION } +### Build matrices ### + variable BASE_BUILD_MATRIX { default = { builds = [ @@ -93,6 +96,7 @@ variable WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX { } } +### Group definitions ### group "default" { targets = [ "product-base", @@ -105,6 +109,53 @@ group "default" { ] } +group "build" { + targets = [ + "product-base", + "product-base-pro", + "connect", + "connect-content-init", + "package-manager", + "r-session-complete", + "workbench", + ] +} + +group "build-test" { + targets = [ + "product-base", + "product-base-pro", + "connect", + "connect-content-init", + "package-manager", + "r-session-complete", + "workbench", + "test-product-base", + "test-product-base-pro", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + "test-connect-content-init", + "test-package-manager", + "test-r-session-complete", + "test-workbench", + "test-workbench-for-google-cloud-workstations", + "test-workbench-for-microsoft-azure-ml", + ] +} + +group "test" { + targets = [ + "test-product-base", + "test-product-base-pro", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + "test-connect-content-init", + "test-package-manager", + "test-r-session-complete", + "test-workbench", + "test-workbench-for-google-cloud-workstations", + "test-workbench-for-microsoft-azure-ml", + ] +} + group "base-images" { targets = [ "product-base", @@ -124,7 +175,7 @@ group "package-manager-images" { group "connect-images" { targets = [ "connect", - "test-connect", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide ] } @@ -142,8 +193,30 @@ group "r-session-complete-images" { ] } -### Base Image targets ### +group "workbench-images" { + targets = [ + "workbench", + "test-workbench", + ] +} + +group "wgcw-images" { + targets = [ + "workbench-for-google-cloud-workstations", + "test-workbench-for-google-cloud-workstations", + ] +} + +group "waml-images" { + targets = [ + "build-workbench-for-microsoft-azure-ml", + "scan-workbench-for-microsoft-azure-ml", + "test-workbench-for-microsoft-azure-ml", + "workbench-for-microsoft-azure-ml", + ] +} +### Base Image targets ### target "base" { labels = { "maintainer" = "Posit Docker " @@ -158,7 +231,7 @@ target "product-base" { tags = [ "ghcr.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", "docker.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + ] dockerfile = "Dockerfile.${builds.os}" context = "product/base" @@ -179,19 +252,13 @@ target "test-product-base" { target = "test" name = "test-product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] - dockerfile = "Dockerfile.${builds.os}" - context = "product/base" + contexts = { + build = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } matrix = BASE_BUILD_MATRIX - args = { - R_VERSION = builds.r_primary - R_VERSION_ALT = builds.r_alternate - PYTHON_VERSION = builds.py_primary - PYTHON_VERSION_ALT = builds.py_alternate - TINI_VERSION = "0.19.0" - QUARTO_VERSION = "1.3.340" - } } target "product-base-pro" { @@ -207,7 +274,7 @@ target "product-base-pro" { dockerfile = "Dockerfile.${builds.os}" context = "product/pro" contexts = { - baseapp = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + product-base = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" } matrix = PRO_BUILD_MATRIX @@ -227,26 +294,18 @@ target "test-product-base-pro" { target = "test" name = "test-product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] - dockerfile = "Dockerfile.${builds.os}" - context = "product/pro" + contexts = { + build = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } matrix = PRO_BUILD_MATRIX - args = { - R_VERSION = builds.r_primary - R_VERSION_ALT = builds.r_alternate - PYTHON_VERSION = builds.py_primary - PYTHON_VERSION_ALT = builds.py_alternate - DRIVERS_VERSION = get_drivers_version(builds.os) - TINI_VERSION = "0.19.0" - QUARTO_VERSION = "1.3.340" - } } ### Package Manager targets ### - target "package-manager" { - inherits = ["product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["base"] target = "build" name = "package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" @@ -258,7 +317,7 @@ target "package-manager" { dockerfile = "Dockerfile.${builds.os}" context = "package-manager" contexts = { - baseapp = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + product-base = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" } matrix = PACKAGE_MANAGER_BUILD_MATRIX @@ -276,36 +335,34 @@ target "test-package-manager" { target = "test" name = "test-package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] - dockerfile = "Dockerfile.${builds.os}" - context = "package-manager" + contexts = { + build = "target:package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } matrix = PACKAGE_MANAGER_BUILD_MATRIX - args = { - R_VERSION = builds.r_primary - R_VERSION_ALT = builds.r_alternate - PYTHON_VERSION = builds.py_primary - PYTHON_VERSION_ALT = builds.py_alternate - RSPM_VERSION = PACKAGE_MANAGER_VERSION - } } ### Connect targets ### - target "connect" { - inherits = ["product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["base"] target = "build" name = "connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [ "ghcr.io/rstudio/rstudio-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", "docker.io/rstudio/rstudio-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + ] + output = [ + "type=cacheonly", + "type=docker", + ] dockerfile = "Dockerfile.${builds.os}" context = "connect" contexts = { - baseapp = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" } matrix = CONNECT_BUILD_MATRIX @@ -324,28 +381,25 @@ target "test-connect" { target = "test" name = "test-connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] - dockerfile = "Dockerfile.${builds.os}" - context = "connect" + contexts = { + build = "target:connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } matrix = CONNECT_BUILD_MATRIX - args = { - R_VERSION = builds.r_primary - R_VERSION_ALT = builds.r_alternate - PYTHON_VERSION = builds.py_primary - PYTHON_VERSION_ALT = builds.py_alternate - RSC_VERSION = CONNECT_VERSION - } } target "connect-content-init" { inherits = ["base"] target = "build" - name = "connect-content-init-${builds.os}" + name = "connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" tags = [ "ghcr.io/rstudio/product-connect-content-init:${builds.os}", + "ghcr.io/rstudio/product-connect-content-init:${builds.os}-${CONNECT_VERSION}", "docker.io/rstudio/product-connect-content-init:${builds.os}", + "docker.io/rstudio/product-connect-content-init:${builds.os}-${CONNECT_VERSION}", ] dockerfile = "Dockerfile.${builds.os}" @@ -359,25 +413,22 @@ target "connect-content-init" { } target "test-connect-content-init" { - inherits = ["connect-content-init-${builds.os}"] + inherits = ["connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}"] target = "test" - name = "test-connect-content-init-${builds.os}" + name = "test-connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + tags = [] - dockerfile = "Dockerfile.${builds.os}" - context = "connect-content-init" + contexts = { + build = "target:connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + } matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX - - args = { - RSC_VERSION = CONNECT_VERSION - } } ### Workbench targets ### - target "r-session-complete" { - inherits = ["product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["base"] target = "build" name = "r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" @@ -389,7 +440,7 @@ target "r-session-complete" { dockerfile = "Dockerfile.${builds.os}" context = "r-session-complete" contexts = { - baseapp = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" } matrix = R_SESSION_COMPLETE_BUILD_MATRIX @@ -410,21 +461,13 @@ target "test-r-session-complete" { target = "test" name = "test-r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] - dockerfile = "Dockerfile.${builds.os}" - context = "r-session-complete" + contexts = { + build = "target:r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } matrix = R_SESSION_COMPLETE_BUILD_MATRIX - args = { - R_VERSION = builds.r_primary - R_VERSION_ALT = builds.r_alternate - PYTHON_VERSION = builds.py_primary - PYTHON_VERSION_ALT = builds.py_alternate - JUPYTERLAB_VERSION = "3.6.5" - RSW_VERSION = WORKBENCH_VERSION - RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" - RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" - } } target "workbench" { @@ -439,7 +482,7 @@ target "workbench" { dockerfile = "Dockerfile.${builds.os}" context = "workbench" contexts = { - baseapp = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" } matrix = WORKBENCH_BUILD_MATRIX @@ -455,15 +498,27 @@ target "workbench" { } } -### Specialty image targets ### +target "test-workbench" { + inherits = ["workbench-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + name = "test-workbench-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] + + contexts = { + build = "target:workbench-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX +} + +### Workbench for Google Cloud Workstations targets ### target "workbench-for-google-cloud-workstations" { inherits = ["base"] name = "workbench-for-google-cloud-workstation-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [ - "ghcr.io/rstudio/product-workbench-for-google-cloud-workstation:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/product-workbench-for-google-cloud-workstation:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "ghcr.io/rstudio/workbench-for-google-cloud-workstation:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/workbench-for-google-cloud-workstation:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", ] dockerfile = "Dockerfile.${builds.os}" @@ -485,19 +540,31 @@ target "workbench-for-google-cloud-workstations" { } } -target "workbench-for-microsoft-azure-ml" { +target "test-workbench-for-google-cloud-workstations" { + inherits = ["workbench-for-google-cloud-workstation-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + + name = "test-workbench-for-google-cloud-workstation-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] + + contexts = { + build = "target:workbench-for-google-cloud-workstation-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX +} + +### Workbench for Microsoft Azure ML targets ### + +target "build-workbench-for-microsoft-azure-ml" { inherits = ["base"] + target = "build" - name = "workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - tags = [ - "ghcr.io/rstudio/product-workbench-for-microsoft-azure-ml:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/product-workbench-for-microsoft-azure-ml:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + name = "build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" dockerfile = "Dockerfile.${builds.os}" context = "workbench-for-microsoft-azure-ml" contexts = { - baseapp = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" } matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX @@ -512,4 +579,48 @@ target "workbench-for-microsoft-azure-ml" { RSW_NAME = "rstudio-workbench" RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" } +} + +target "scan-workbench-for-microsoft-azure-ml" { + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "clamav" + + name = "scan-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + + contexts = { + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX +} + +target "test-workbench-for-microsoft-azure-ml" { + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "test-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + + contexts = { + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX +} + +target "workbench-for-microsoft-azure-ml" { + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "final" + + name = "workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-workbench-for-microsoft-azure-ml:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-workbench-for-microsoft-azure-ml:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + contexts = { + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + clamav = "target:scan-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX } \ No newline at end of file diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204 index fd5d4efe2..d3bc84741 100644 --- a/package-manager/Dockerfile.ubuntu2204 +++ b/package-manager/Dockerfile.ubuntu2204 @@ -4,8 +4,8 @@ ARG PYTHON_VERSION=3.12.1 ARG PYTHON_VERSION_ALT=3.11.7 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build -LABEL maintainer="RStudio Docker " +FROM product-base as build +LABEL maintainer="Posit Docker " ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 diff --git a/product/pro/Dockerfile.centos7 b/product/pro/Dockerfile.centos7 index a0ed998e7..98ba632b7 100644 --- a/product/pro/Dockerfile.centos7 +++ b/product/pro/Dockerfile.centos7 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.14 ARG PYTHON_VERSION_ALT=3.8.15 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build +FROM product-base as build LABEL maintainer="Posit Docker " ARG R_VERSION=4.2.0 @@ -34,6 +34,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG TINI_VERSION=0.19.0 ARG QUARTO_VERSION=1.3.340 +ARG DRIVERS_VERSION=2024.03.0-1 ARG OS=centos7 COPY test/ /tmp/test diff --git a/product/pro/Dockerfile.ubuntu2204 b/product/pro/Dockerfile.ubuntu2204 index d9335dd7e..b78a1aeae 100644 --- a/product/pro/Dockerfile.ubuntu2204 +++ b/product/pro/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build +FROM product-base as build LABEL maintainer="Posit Docker " ARG DEBIAN_FRONTEND=noninteractive @@ -37,6 +37,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG TINI_VERSION=0.19.0 ARG QUARTO_VERSION=1.3.340 +ARG DRIVERS_VERSION=2024.03.0 ARG OS=ubuntu2204 RUN --mount=type=bind,target=/tmp/test,source=test/ \ diff --git a/r-session-complete/Dockerfile.ubuntu2204 b/r-session-complete/Dockerfile.ubuntu2204 index 1a27dd09a..7e3f02060 100644 --- a/r-session-complete/Dockerfile.ubuntu2204 +++ b/r-session-complete/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build +FROM product-base-pro as build LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive diff --git a/r-session-complete/test/run_tests.sh b/r-session-complete/test/run_tests.sh index 139739f34..0c71dcad3 100755 --- a/r-session-complete/test/run_tests.sh +++ b/r-session-complete/test/run_tests.sh @@ -1,7 +1,5 @@ #!/bin/bash -# install goss - GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 index b612a6a97..e7211a2fc 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 @@ -1,4 +1,4 @@ -FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:public-image-current +FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:public-image-current as build ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive @@ -147,8 +147,7 @@ RUN /opt/python/"${PYTHON_VERSION_JUPYTER}"/bin/python -m venv /opt/python/jupyt && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge -RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ - && chmod +x /usr/local/bin/wait-for-it.sh +ADD --chmod=755 https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh /usr/local/bin/wait-for-it.sh RUN mkdir -p /var/lib/rstudio-server/monitor/log \ && chown -R rstudio-server:rstudio-server /var/lib/rstudio-server/monitor \ @@ -179,3 +178,22 @@ RUN apt-get remove -yq dpkg-sig \ EXPOSE 80/tcp EXPOSE 5559/tcp + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.11.7 +ARG PYTHON_VERSION_ALT=3.10.13 +ARG PYTHON_VERSION_JUPYTER=3.10.13 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG QUARTO_VERSION=1.3.450 +ARG DRIVERS_VERSION=2023.05.0 +ARG RSW_VERSION=2023.12.1+402.pro1 +ARG DOCKER_TEST_STAGE=true + +COPY deps/py_packages.txt /tmp/py_packages.txt +COPY deps/r_packages.txt /tmp/r_packages.txt +RUN apt-get update && apt-get install -yq net-tools +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index 674665ef4..e2d692fad 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -35,19 +35,14 @@ port: tcp:80: listening: true ip: - - 0.0.0.0 - skip: false - tcp:5559: - listening: true - ip: - - 127.0.0.1 + - 0.0.0.0 skip: false process: rserver: running: true skip: false - rstudio-launche: + rserver-launche: running: true skip: false @@ -88,14 +83,14 @@ file: exists: true /etc/rstudio/jupyter.conf: exists: true - contains: + contents: - --LabApp.token="" - --no-browser - --allow-root - --ip=0.0.0.0 /etc/rstudio/rserver.conf: exists: true - contains: + contents: - www-port=80 - launcher-sessions-callback-address=http://0.0.0.0:80 - auth-proxy=1 @@ -103,21 +98,21 @@ file: - auth-proxy-user-header=x-custom-user-name /etc/rstudio/nginx.site.conf: exists: true - contains: + contents: - proxy_set_header X-CUSTOM-USER-NAME user/google; - proxy_set_header Host $http_x_forwarded_host; /etc/rstudio/vscode.extensions.conf: exists: true - contains: + contents: - quarto.quarto - GoogleCloudTools.cloudcode /tmp/startup.log: exists: true - contains: + contents: - "!Error reading /etc/rstudio/rserver.conf:" /etc/pam.d/common-session: exists: true - contains: + contents: - "/^session required pam_mkhomedir.so skel=/etc/skel umask=0022$/" /etc/sssd/sssd.conf: exists: true @@ -135,7 +130,7 @@ file: group: root mode: "0755" filetype: file - contains: [ + contents: [ "useradd -m user" ] /etc/workstation-startup.d/110_config-jupyter.sh: @@ -144,7 +139,7 @@ file: group: root mode: "0755" filetype: file - contains: [ + contents: [ # Checks that we're setting the Jupyter shell to /bin/bash "echo \"c.ServerApp.terminado_settings = {'shell_command': ['/bin/bash']}\" > /home/user/.jupyter/jupyter_notebook_config.py" ] @@ -154,7 +149,7 @@ file: group: root mode: "0755" filetype: file - contains: [ + contents: [ "/usr/bin/supervisord -c /etc/supervisor/supervisord.conf" ] diff --git a/workbench-for-google-cloud-workstations/test/run_tests.sh b/workbench-for-google-cloud-workstations/test/run_tests.sh index 02c2cd04f..dd9d1b236 100755 --- a/workbench-for-google-cloud-workstations/test/run_tests.sh +++ b/workbench-for-google-cloud-workstations/test/run_tests.sh @@ -9,25 +9,22 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & +sleep 10 + +netstat -tulpn echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:80 -t $RSW_TIMEOUT wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.22} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 index 4a8156945..b964c5f37 100644 --- a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 +++ b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} AS workbench +FROM product-base-pro AS build LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive @@ -115,8 +115,7 @@ RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install -r /tmp/py_packages && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \ && rm /tmp/py_packages.txt -RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ - && chmod +x /usr/local/bin/wait-for-it.sh +ADD --chmod=755 https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh /usr/local/bin/wait-for-it.sh RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash \ && az extension add -n ml -y @@ -130,9 +129,22 @@ EXPOSE 5559/tcp ENTRYPOINT [] CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] +FROM build AS test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG PYTHON_VERSION_JUPYTER=3.8.17 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG RSW_VERSION=2023.12.1+402.pro1 +ARG DOCKER_TEST_STAGE=true + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh -FROM workbench AS clamav +FROM build AS clamav # Set up ClamAV RUN apt-get update \ @@ -169,7 +181,7 @@ RUN clamscan \ # with ClamAV installed. Avoid adding changes in this stage unless they are related # to the ClamAV stage. Since the ClamAV stage may be cached, you can't guarantee # another full scan if you change anything beyond this point. -FROM workbench AS final +FROM build AS final # Copy ClamAV scan logs so the end user can see them. COPY --from=clamav /var/log/clamav/clamscan.log /var/log/clamav/clamupdate.log / diff --git a/workbench-for-microsoft-azure-ml/test/goss_vars.yaml b/workbench-for-microsoft-azure-ml/test/goss_vars.yaml index 2669ba18b..65f2d26c5 100644 --- a/workbench-for-microsoft-azure-ml/test/goss_vars.yaml +++ b/workbench-for-microsoft-azure-ml/test/goss_vars.yaml @@ -19,7 +19,6 @@ syspkgs: - libglib2.0-0 - libpq5 - libsm6 - - libssl-dev - openssl - libssl-dev - libuser diff --git a/workbench-for-microsoft-azure-ml/test/run_tests.sh b/workbench-for-microsoft-azure-ml/test/run_tests.sh index 9dc299cb2..daa29c97f 100755 --- a/workbench-for-microsoft-azure-ml/test/run_tests.sh +++ b/workbench-for-microsoft-azure-ml/test/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash set -xe -RSW_TIMEOUT=${RSW_TIMEOUT:-15} +RSW_TIMEOUT=${RSW_TIMEOUT:-60} touch /tmp/startup.log trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/startup.log; exit $err' ERR @@ -9,24 +9,21 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & +sleep 15 echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT +wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VARS_FILE=${GOSS_VARS_FILE:-/tmp/test/goss_vars.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT diff --git a/workbench/Dockerfile.ubuntu2204 b/workbench/Dockerfile.ubuntu2204 index 64edfed12..7511c72ba 100644 --- a/workbench/Dockerfile.ubuntu2204 +++ b/workbench/Dockerfile.ubuntu2204 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} +FROM product-base-pro as build LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive @@ -108,3 +108,19 @@ EXPOSE 5559/tcp ENTRYPOINT [] CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] + +FROM build as test + +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG PYTHON_VERSION_JUPYTER=3.8.17 +ARG JUPYTERLAB_VERSION=3.6.5 +ARG RSW_VERSION=2023.12.1+402.pro1 +ARG DOCKER_TEST_STAGE=true + +RUN --mount=type=bind,target=/tmp/test,source=test/ \ + /tmp/test/run_tests.sh + + diff --git a/workbench/test/goss.yaml b/workbench/test/goss.yaml index 0a9a78c70..b48143786 100644 --- a/workbench/test/goss.yaml +++ b/workbench/test/goss.yaml @@ -78,6 +78,7 @@ file: exists: true {{ end }} /var/lib/rstudio-launcher/Local/jobs/buildkitsandbox: + skip: {{ .Env.DOCKER_TEST_STAGE }} # This file is present during the build stage, but not after exists: false /etc/rstudio/vscode.conf: exists: true diff --git a/workbench/test/run_tests.sh b/workbench/test/run_tests.sh index 9dc299cb2..76433623f 100755 --- a/workbench/test/run_tests.sh +++ b/workbench/test/run_tests.sh @@ -14,19 +14,13 @@ echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT echo "--> Startup complete" -GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} -GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} -# default to empty var file (since vars are not necessary) -if [ ! -f "$GOSS_VARS" ]; then - touch $GOSS_VARS -fi - # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ && chmod +x /tmp/goss \ && GOSS=/tmp/goss -GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT +GOSS_FILE=$GOSS_FILE $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT From 9fefe59b72e459930b18549dd0e9e5bc6c28569e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 11:09:54 -0700 Subject: [PATCH 11/37] Implement separate test for Connect --- .github/workflows/build-bake.yaml | 42 ++- .github/workflows/build-content.yaml | 215 ------------ .github/workflows/build-prerelease.yaml | 144 --------- .github/workflows/build-release.yaml | 360 --------------------- .github/workflows/build-workbench-aml.yaml | 127 -------- .github/workflows/build-workbench-gcw.yaml | 99 ------ .gitignore | 2 + build.justfile | 53 +++ docker-bake.hcl | 12 +- share/local_buildkitd.toml | 2 + tools/bake_test_command_extract.py | 64 ++++ 11 files changed, 162 insertions(+), 958 deletions(-) delete mode 100644 .github/workflows/build-content.yaml delete mode 100644 .github/workflows/build-prerelease.yaml delete mode 100644 .github/workflows/build-release.yaml delete mode 100644 .github/workflows/build-workbench-aml.yaml delete mode 100644 .github/workflows/build-workbench-gcw.yaml create mode 100755 build.justfile create mode 100644 share/local_buildkitd.toml create mode 100644 tools/bake_test_command_extract.py diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index fb7c2a17c..25c396d94 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -5,20 +5,40 @@ on: - dev pull_request: -name: Build Docker Images with Bake +name: Bake Docker Images jobs: build: runs-on: ubuntu-latest-4x steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push - uses: docker/bake-action@v4 - with: - targets: "build-test" - push: false + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/bake-action@v4 + with: + targets: "build-test" + push: false + + - name: Test Connect (privileged container) + run: | + just -f build.justfile test-connect + + - name: Build and push + if: github.ref == 'refs/heads/main' + uses: docker/bake-action@v4 + with: + targets: "build" + push: true diff --git a/.github/workflows/build-content.yaml b/.github/workflows/build-content.yaml deleted file mode 100644 index 4b7db4e1c..000000000 --- a/.github/workflows/build-content.yaml +++ /dev/null @@ -1,215 +0,0 @@ -on: - push: - branches: - - main - - dev - pull_request: - -name: Content Images - Build, Test, Scan, and Push -jobs: - matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - - id: set-matrix - run: | - MATRIX=$(jq -Mcr < content/matrix.json) - echo "matrix=$MATRIX" >> $GITHUB_OUTPUT - - build: - runs-on: ubuntu-latest - needs: matrix - name: content-base-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}--${{ github.ref }} - - permissions: - contents: read - packages: write - - concurrency: - group: content-base-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ github.ref }} - cancel-in-progress: true - - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.matrix.outputs.matrix) }} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - get-content-args \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - get-content-tags \ - content-base \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ${{ matrix.config.os }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push content base image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./content/base - os: ${{ matrix.config.os }} - product: content-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./content/base - os: ${{ matrix.config.os }} - product: content-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic - - build-pro: - runs-on: ubuntu-latest - needs: [ matrix, build ] - name: content-pro-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}--${{ github.ref }} - concurrency: - group: content-pro-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ github.ref }} - cancel-in-progress: true - - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.matrix.outputs.matrix) }} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - get-content-args \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ${{ matrix.config.drivers }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - get-content-tags \ - content-pro \ - ${{ matrix.config.r }} \ - ${{ matrix.config.py }} \ - ${{ matrix.config.os }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push content pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./content/pro - os: ${{ matrix.config.os }} - product: content-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./content/pro - os: ${{ matrix.config.os }} - product: content-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - test-image: false - push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml deleted file mode 100644 index 470470bb9..000000000 --- a/.github/workflows/build-prerelease.yaml +++ /dev/null @@ -1,144 +0,0 @@ -on: - schedule: - # every morning at 8am UTC - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule - # make sure to change the `if` commands below when changing the schedule - - cron: '0 8 * * *' - - cron: '0 9 * * *' - push: - branches: - - main - - dev - - dev-rspm - pull_request: - -name: Prerelease - Build, Test, Scan, and Push -jobs: - build: - runs-on: ubuntu-latest - name: build-${{ matrix.config.type }}-${{ matrix.config.product }}-${{ matrix.config.os }} - - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: - config: - - {product: "workbench", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "workbench", type: "preview", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "connect", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "connect-content-init", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "package-manager", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - {product: "r-session-complete", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "r-session-complete", type: "daily", os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: "r-session-complete", type: "preview", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: "r-session-complete", type: "preview", os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - concurrency: - group: prerelease-build-${{ matrix.config.type }}-${{ matrix.config.product }}-${{ matrix.config.os }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Check Out main Branch - if: github.event.schedule == '0 8 * * *' - uses: actions/checkout@v3 - with: - ref: 'main' - - - name: Check Out Repo at Triggered Branch - if: github.event.schedule != '0 8 * * *' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=$(just -f ci.Justfile get-version ${{ matrix.config.product }} --type=${{ matrix.config.type }} --local) - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-prerelease-args \ - ${{ matrix.config.type }} \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-prerelease-tags \ - ${{ matrix.config.type }} \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: ${{ matrix.config.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: product-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml deleted file mode 100644 index 72eb2cc5b..000000000 --- a/.github/workflows/build-release.yaml +++ /dev/null @@ -1,360 +0,0 @@ -on: - schedule: - - cron: '0 12 * * 1' # If updating this value, be sure to update logic for all `push-image` arguments! - push: - branches: - - main - - dev - pull_request: - -name: Release - Build, Test, Scan, and Push -jobs: - build-base: - runs-on: ubuntu-latest - name: product-base-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }} - - permissions: - contents: read - packages: write - - concurrency: - group: base-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }}-${{ github.ref }} - cancel-in-progress: true - - strategy: - fail-fast: false - matrix: - config: - - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 12 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 12 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-args ${{ matrix.config.os }} product-base \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-tags ${{ matrix.config.os }} product-base \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./product/base - os: ${{ matrix.config.os }} - product: product-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./product/base - os: ${{ matrix.config.os }} - product: product-base - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic - - build-base-pro: - needs: build-base - runs-on: ubuntu-latest - name: product-base-pro-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }} - - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: - config: - - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - concurrency: - group: build-base-pro-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 12 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 12 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-args ${{ matrix.config.os }} product-base-pro \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-base-tags ${{ matrix.config.os }} product-base-pro \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./product/pro - os: ${{ matrix.config.os }} - product: product-base-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./product/pro - os: ${{ matrix.config.os }} - product: product-base-pro - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic - - build-products: - needs: [ build-base, build-base-pro ] - runs-on: ubuntu-latest - name: build-${{ matrix.config.product }}-${{ matrix.config.os }} - - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: - config: - - {product: 'workbench', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'connect', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: 'connect-content-init', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.12.1", py-alternate: "3.11.7"} - - {product: 'r-session-complete', os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'r-session-complete', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - concurrency: - group: build-products-${{ matrix.config.product }}-${{ matrix.config.os }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 12 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 12 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=`just -f ci.Justfile get-version ${{ matrix.config.product }} --type=release --local` - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-product-args \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ matrix.config.r-primary }} \ - R_VERSION_ALT=${{ matrix.config.r-alternate }} \ - PYTHON_VERSION=${{ matrix.config.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ - get-product-tags \ - ${{ matrix.config.product }} \ - ${{ matrix.config.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push product image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: ${{ matrix.config.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push product image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./${{ matrix.config.product }} - os: ${{ matrix.config.os }} - product: ${{ matrix.config.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 12 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-workbench-aml.yaml b/.github/workflows/build-workbench-aml.yaml deleted file mode 100644 index 9e70c1490..000000000 --- a/.github/workflows/build-workbench-aml.yaml +++ /dev/null @@ -1,127 +0,0 @@ -on: - schedule: - - cron: '0 14 * * 1' # If updating this value, be sure to update logic for all `push-image` arguments! - push: - branches: - - main - - dev - pull_request: - -name: Workbench for Azure ML - Build, Test, Scan, and Push -jobs: - build-workbench-for-azure-ml: - env: - product: workbench-for-microsoft-azure-ml - os: ubuntu2204 - r-primary: 4.2.3 - r-alternate: 4.1.3 - py-primary: 3.9.14 - py-alternate: 3.8.15 - runs-on: ubuntu-latest-4x - name: build-workbench-for-azure-ml - - permissions: - contents: read - packages: write - - steps: - - - name: Check Out Repo - cron main - if: github.event.schedule == '0 14 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 14 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=$(just -f ci.Justfile get-version ${{ env.product }} --type=release --local) - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ env.r-primary }} \ - R_VERSION_ALT=${{ env.r-alternate }} \ - PYTHON_VERSION=${{ env.py-primary }} \ - PYTHON_VERSION_ALT=${{ env.py-alternate }} \ - get-product-args \ - ${{ env.product }} \ - ${{ env.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - R_VERSION=${{ env.r-primary }} \ - R_VERSION_ALT=${{ env.r-alternate }} \ - PYTHON_VERSION=${{ env.py-primary }} \ - PYTHON_VERSION_ALT=${{ env.py-alternate }} \ - get-product-tags \ - ${{ env.product }} \ - ${{ env.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push product image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./${{ env.product }} - os: ${{ env.os }} - product: ${{ env.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push product image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./${{ env.product }} - os: ${{ env.os }} - product: ${{ env.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.github/workflows/build-workbench-gcw.yaml b/.github/workflows/build-workbench-gcw.yaml deleted file mode 100644 index 750ae7bd4..000000000 --- a/.github/workflows/build-workbench-gcw.yaml +++ /dev/null @@ -1,99 +0,0 @@ -on: - schedule: - - cron: '0 14 * * 1' # If updating this value, be sure to update logic for all `push-image` arguments! - push: - branches: - - main - - dev - pull_request: - -name: Workbench for GCW - Build, Test, Scan, and Push -jobs: - build-workbench-for-google-cloud-workstations: - runs-on: ubuntu-latest-4x - name: build-workbench-for-google-cloud-workstations - - permissions: - contents: read - packages: write - - steps: - - name: Check Out Repo - cron main - if: github.event.schedule == '0 14 * * 1' - uses: actions/checkout@v3 - with: - ref: main - - - name: Check Out Repo - if: github.event.schedule != '0 14 * * 1' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Version - id: get-version - run: | - VERSION=$(just workbench-for-google-cloud-workstations/get-version) - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - BUILD_ARGS=$(just workbench-for-google-cloud-workstations/get-build-args) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - IMAGE_TAGS=$(just workbench-for-google-cloud-workstations/get-build-tags) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - - name: Build/Test/Scan/Push base pro image - id: build1 - uses: ./.github/actions/build-test-scan-push - continue-on-error: true - with: - context: ./workbench-for-google-cloud-workstations - os: ubuntu2004 - product: workbench-for-google-cloud-workstations - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # Begin retry logic - - - name: Wait 60s on failure before retrying - if: steps.build1.outcome == 'failure' - run: sleep 60 - - - name: Retry - Build/Test/Scan/Push base pro image - id: build2 - if: steps.build1.outcome == 'failure' - uses: ./.github/actions/build-test-scan-push - with: - context: ./workbench-for-google-cloud-workstations - os: ubuntu2004 - product: workbench-for-google-cloud-workstations - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.event.schedule == '0 14 * * 1' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.GITHUB_TOKEN }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - # End retry logic diff --git a/.gitignore b/.gitignore index c9c09d208..f2b8f9669 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ data .idea/ tmp*/ .python-version +.out/ +.cache/ diff --git a/build.justfile b/build.justfile new file mode 100755 index 000000000..1467f3c48 --- /dev/null +++ b/build.justfile @@ -0,0 +1,53 @@ +#!/usr/bin/env just --justfile + +# Targets for managing the buildx builder for Posit images + +# just create-builder +create-builder: + docker buildx create \ + --name posit-builder \ + --buildkitd-flags '--allow-insecure-entitlement security.insecure' \ + --buildkitd-config "{{justfile_directory()}}/share/local_buildkitd.toml" + +# just delete-builder +delete-builder: + docker buildx rm posit-builder + +# Build and bake + +# just bake connect +bake target: + just -f {{justfile()}} create-builder || true + docker buildx bake --builder=posit-builder -f docker-bake.hcl {{target}} + +# just plan +plan: + docker buildx bake -f docker-bake.hcl --print + +# just build +build: + just -f {{justfile()}} bake build + +# just test +test: + just -f {{justfile()}} bake test + just -f {{justfile()}} test-connect ubuntu2204 + +# just build-test +build-test: + just -f {{justfile()}} build + just -f {{justfile()}} test + +# Run tests + +# just test-connect ubuntu2204 +test-connect os="ubuntu2204": + #!/bin/bash + just -f {{justfile()}} create-builder || true + build_cmd=$(python3 {{justfile_directory()}}/tools/bake_test_command_extract.py connect {{os}}) + echo $build_cmd + if [[ $? -ne 0 ]]; then + echo "Failed to extract build command" + exit 1 + fi + bash -c "$build_cmd" diff --git a/docker-bake.hcl b/docker-bake.hcl index a03e7de83..403544935 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -221,6 +221,7 @@ target "base" { labels = { "maintainer" = "Posit Docker " } + output = ["type=cacheonly"] } target "product-base" { @@ -232,6 +233,9 @@ target "product-base" { "ghcr.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", "docker.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", ] + output = [ + "type=cacheonly", + ] dockerfile = "Dockerfile.${builds.os}" context = "product/base" @@ -269,7 +273,10 @@ target "product-base-pro" { tags = [ "ghcr.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", "docker.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + ] + output = [ + "type=cacheonly", + ] dockerfile = "Dockerfile.${builds.os}" context = "product/pro" @@ -354,9 +361,10 @@ target "connect" { "ghcr.io/rstudio/rstudio-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", "docker.io/rstudio/rstudio-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", ] + # We output Connect to OCI so it can be pulled in for testing later on. output = [ "type=cacheonly", - "type=docker", + "type=oci,tar=false,dest=./.out/connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" ] dockerfile = "Dockerfile.${builds.os}" diff --git a/share/local_buildkitd.toml b/share/local_buildkitd.toml new file mode 100644 index 000000000..531aab30f --- /dev/null +++ b/share/local_buildkitd.toml @@ -0,0 +1,2 @@ +[worker.oci] + max-parallelism = 4 \ No newline at end of file diff --git a/tools/bake_test_command_extract.py b/tools/bake_test_command_extract.py new file mode 100644 index 000000000..733efd7a8 --- /dev/null +++ b/tools/bake_test_command_extract.py @@ -0,0 +1,64 @@ +import argparse +import json +import subprocess +from pathlib import Path + +PROJECT_DIR = Path(__file__).resolve().parents[1] + + +parser = argparse.ArgumentParser( + description="Extract a test command from a bake plan" +) +parser.add_argument("image_name") +parser.add_argument("operating_system") +parser.add_argument("--builder", default="posit-builder") + + +def get_bake_plan(): + cmd = ["docker", "buildx", "bake", "-f", str(PROJECT_DIR / "docker-bake.hcl"), "--print"] + p = subprocess.run(cmd, capture_output=True) + if p.returncode != 0: + print(f"Failed to get bake plan: {p.stderr}") + exit(1) + return json.loads(p.stdout.decode("utf-8")) + + +def get_targets(plan, image_name, operating_system): + targets = {} + for target_name, target_spec in plan["target"].items(): + if target_name.startswith(f"{image_name}-{operating_system}"): + targets[target_name] = target_spec + return targets + + +def build_test_command(target_name, target_spec, builder): + cmd = [ + "docker", + "buildx", + "--builder", + builder, + "build", + "--allow", + "security.insecure", + "--target", + "test", + ] + cmd.extend(["--build-context", f"build=oci-layout://{PROJECT_DIR / '.out' / target_name}"]) + for name, value in target_spec["args"].items(): + cmd.extend(["--build-arg", f'{name}="{value}"']) + cmd.extend(["--file", str(PROJECT_DIR / target_spec["context"] / target_spec["dockerfile"])]) + cmd.append(str(PROJECT_DIR / target_spec["context"])) + return cmd + + +def main(): + args = parser.parse_args() + plan = get_bake_plan() + targets = get_targets(plan, args.image_name, args.operating_system) + for target_name, target_spec in targets.items(): + cmd = build_test_command(target_name, target_spec, args.builder) + print(" ".join(cmd)) + + +if __name__ == "__main__": + main() From 9f263bbdf088871edfce7c9a1c829daf802d1544 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 11:14:00 -0700 Subject: [PATCH 12/37] Remove compose testing and image local justfiles --- connect-content-init/Justfile | 50 -------- connect-content-init/docker-compose.test.yml | 13 -- connect/Justfile | 83 ------------- connect/docker-compose.test.yml | 20 --- package-manager/Justfile | 98 --------------- package-manager/docker-compose.test.yml | 19 --- product/base/Justfile | 69 ----------- product/base/docker-compose.test.yml | 20 --- product/pro/Justfile | 79 ------------ product/pro/docker-compose.test.yml | 22 ---- r-session-complete/Justfile | 87 ------------- .../Justfile | 116 ------------------ .../docker-compose.test.yml | 24 ---- workbench-for-microsoft-azure-ml/Justfile | 78 ------------ .../docker-compose.test.yml | 20 --- workbench/Justfile | 100 --------------- workbench/docker-compose.test.yml | 20 --- 17 files changed, 918 deletions(-) delete mode 100644 connect-content-init/Justfile delete mode 100644 connect-content-init/docker-compose.test.yml delete mode 100644 connect/Justfile delete mode 100644 connect/docker-compose.test.yml delete mode 100644 package-manager/Justfile delete mode 100644 package-manager/docker-compose.test.yml delete mode 100755 product/base/Justfile delete mode 100644 product/base/docker-compose.test.yml delete mode 100644 product/pro/Justfile delete mode 100644 product/pro/docker-compose.test.yml delete mode 100755 r-session-complete/Justfile delete mode 100644 workbench-for-google-cloud-workstations/Justfile delete mode 100644 workbench-for-google-cloud-workstations/docker-compose.test.yml delete mode 100644 workbench-for-microsoft-azure-ml/Justfile delete mode 100644 workbench-for-microsoft-azure-ml/docker-compose.test.yml delete mode 100644 workbench/Justfile delete mode 100644 workbench/docker-compose.test.yml diff --git a/connect-content-init/Justfile b/connect-content-init/Justfile deleted file mode 100644 index 3c59f2c92..000000000 --- a/connect-content-init/Justfile +++ /dev/null @@ -1,50 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "connect-content-init" -IMAGE_OS := "ubuntu2204" - -RSC_VERSION := "2024.02.0" -RSC_TAG_SAFE_VERSION := replace(RSC_VERSION, "+", "-") -RSC_LICENSE := "" - -R_VERSION := "3.6.2" -R_VERSION_ALT := "4.1.0" - -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" - -DEFAULT_TAG := IMAGE_PREFIX + PRODUCT + ":" + IMAGE_OS + "-" + RSC_TAG_SAFE_VERSION - -# Build Connect content init image - just build ubuntu2204 2022.10.0 rstudio/rstudio-connect-content-init:ubuntu2204-2022.10.0 -build OS=IMAGE_OS VERSION=RSC_VERSION +TAGS=DEFAULT_TAG: - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - tag_array=() - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSC_VERSION="{{ VERSION }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Connect content init image - just test rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test TAG=DEFAULT_TAG VERSION=RSC_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSC_VERSION="{{ VERSION }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Connect content init image interactively - just test-i rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test-i TAG=DEFAULT_TAG VERSION=RSC_VERSION: - just test {{ TAG }} {{ VERSION }} bash diff --git a/connect-content-init/docker-compose.test.yml b/connect-content-init/docker-compose.test.yml deleted file mode 100644 index 0ee6998b4..000000000 --- a/connect-content-init/docker-compose.test.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSC_VERSION - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/connect/Justfile b/connect/Justfile deleted file mode 100644 index 185940cca..000000000 --- a/connect/Justfile +++ /dev/null @@ -1,83 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "connect" -IMAGE_OS := "ubuntu2204" - -RSC_VERSION := "2024.02.0" -RSC_LICENSE := "" -RSC_LICENSE_SERVER := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -PERSIST_LICENSE := "false" -PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") - -DEFAULT_TAG := IMAGE_PREFIX + PRODUCT + ":" + IMAGE_OS + "-" + RSC_VERSION - -# Build Connect image - just build ubuntu2204 2022.10.0 rstudio/rstudio-connect:ubuntu2204-2022.10.0 -build OS=IMAGE_OS VERSION=RSC_VERSION +TAGS=DEFAULT_TAG: - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - tag_array=() - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSC_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Connect image - just test rstudio/rstudio-connect:ubuntu2204-2022.10.0 2022.10.0 -test OS="ubuntu2204" VERSION=RSC_VERSION: - #!/usr/bin/env bash - docker buildx --builder="{{ BUILDX_PATH }}" build ${BUILDX_ARGS} --allow security.insecure --target test \ - --build-arg RSC_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Run Connect - just RSC_LICENSE="" run rstudio/rstudio-connect:ubuntu2204-2022.10.0 -run TAG=DEFAULT_TAG CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSC_LICENSE }}" ] && [ -z "{{ RSC_LICENSE_SERVER }}" ]; then - echo "Please set RSC_LICENSE or RSC_LICENSE_SERVER before running." - exit 1 - fi - - volume_opts=() - if [ {{ PERSIST_LICENSE }} = "true" ]; then - if [ {{RSC_LICENSE}} ]; then - echo "Volumes will be configured to persist license state data for an activation key." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes key {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - elif [ {{RSC_LICENSE_SERVER}} ]; then - echo "Volumes will be configured to persist license state data for a floating license server." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes float {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - fi - fi - - docker run -it --privileged \ - ${volume_opts[@]} \ - -p 3939:3939 \ - -e RSC_LICENSE="{{ RSC_LICENSE }}" \ - -e RSC_LICENSE_SERVER="{{ RSC_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/connect/docker-compose.test.yml b/connect/docker-compose.test.yml deleted file mode 100644 index 7ac851a5e..000000000 --- a/connect/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - privileged: true - environment: - # uses .env by default - - RSC_VERSION - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - RSC_LICENSE - - RSC_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/package-manager/Justfile b/package-manager/Justfile deleted file mode 100644 index cb03a2f40..000000000 --- a/package-manager/Justfile +++ /dev/null @@ -1,98 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "package-manager" -IMAGE_OS := "ubuntu2204" - -RSPM_VERSION := "2023.12.0-13" -RSPM_LICENSE := "" -RSPM_LICENSE_SERVER := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.12.1" -PYTHON_VERSION_ALT := "3.11.7" - -PERSIST_LICENSE := "false" -PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSPM_VERSION}})" - -# Build Package Manager image - just build ubuntu2204 2022.07.2-11 rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 -build OS=IMAGE_OS VERSION=RSPM_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSPM_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Package Manager image - just test rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 2022.07.2-11 -test TAG=`just _make-default-tag` VERSION=RSPM_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSPM_VERSION="{{ VERSION }}" \ - RSPM_LICENSE="{{ RSPM_LICENSE }}" \ - RSPM_LICENSE_SERVER="{{ RSPM_LICENSE_SERVER }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Package Manager image interactively - just test-i rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 2022.07.2-11 -test-i TAG=`just _make-default-tag` VERSION=RSPM_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Package Manager - just RSPM_LICENSE="" run rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSPM_LICENSE }}" ] && [ -z "{{ RSPM_LICENSE_SERVER }}" ]; then - echo "Please set RSPM_LICENSE or RSPM_LICENSE_SERVER before running." - exit 1 - fi - - volume_opts=() - if [ {{ PERSIST_LICENSE }} = "true" ]; then - if [ {{RSPM_LICENSE}} ]; then - echo "Volumes will be configured to persist license state data for an activation key." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes key {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - elif [ {{RSPM_LICENSE_SERVER}} ]; then - echo "Volumes will be configured to persist license state data for a floating license server." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes float {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - fi - fi - - docker run -it \ - ${volume_opts[@]} \ - -p 4242:4242 \ - -e RSPM_LICENSE="{{ RSPM_LICENSE }}" \ - -e RSPM_LICENSE_SERVER="{{ RSPM_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/package-manager/docker-compose.test.yml b/package-manager/docker-compose.test.yml deleted file mode 100644 index d1279b981..000000000 --- a/package-manager/docker-compose.test.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSPM_VERSION - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - RSPM_LICENSE - - RSPM_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/product/base/Justfile b/product/base/Justfile deleted file mode 100755 index 00c603788..000000000 --- a/product/base/Justfile +++ /dev/null @@ -1,69 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -PRODUCT := "product-base" -IMAGE_OS := "ubuntu2204" - -IMAGE_REGISTRY := "rstudio" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -TINI_VERSION := "0.19.0" -QUARTO_VERSION := "1.3.340" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_REGISTRY}}/{{PRODUCT}}:{{OS}}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}}" - -# Build base image - just build ubuntu2204 rstudio/product-base:ubuntu2204 -build OS=IMAGE_OS *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for TAG in {{TAGS}} - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg TINI_VERSION="{{ TINI_VERSION }}" \ - --build-arg QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - --file=./Dockerfile.$(just -f ../../Justfile _parse-os {{OS}}) . - -# Test base image - just test rstudio/product-base:ubuntu2204 -test TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - TINI_VERSION="{{ TINI_VERSION }}" \ - OS="{{ IMAGE_OS }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test base image interactively - just test-i rstudio/product-base:ubuntu2204 -test-i TAG=`just _make-default-tag`: - just test {{ TAG }} bash diff --git a/product/base/docker-compose.test.yml b/product/base/docker-compose.test.yml deleted file mode 100644 index bbd42874b..000000000 --- a/product/base/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - privileged: true - environment: - # uses .env by default - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - TINI_VERSION - - QUARTO_VERSION - - OS - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/product/pro/Justfile b/product/pro/Justfile deleted file mode 100644 index d07474f01..000000000 --- a/product/pro/Justfile +++ /dev/null @@ -1,79 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -PRODUCT := "product-base-pro" -IMAGE_OS := "ubuntu2204" - -IMAGE_REGISTRY := "rstudio" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -TINI_VERSION := "0.19.0" -QUARTO_VERSION := "1.3.340" - -DRIVERS_VERSION := "2024.03.0" -DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_REGISTRY}}/{{PRODUCT}}:{{OS}}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}}" - -# Build base pro image - just build ubuntu2204 rstudio/product-base-pro:ubuntu2204 -build OS=IMAGE_OS *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - if [[ "{{ OS }}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - tag_array=() - for TAG in $raw_tag_array - do - tag_array+=("-t" $TAG) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg DRIVERS_VERSION="${_DRIVERS_VERSION}" \ - --file=./Dockerfile.$(just -f ../../Justfile _parse-os {{OS}}) . - -# Test base image - just test rstudio/product-base-pro:ubuntu2204 -test TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - - IMAGE_NAME="{{ TAG }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - TINI_VERSION="{{ TINI_VERSION }}" \ - DRIVERS_VERSION="{{ DRIVERS_VERSION }}" \ - OS="{{ IMAGE_OS }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test base image interactively - just test-i rstudio/product-base-pro:ubuntu2204 -test-i TAG=`just _make-default-tag`: - just test {{ TAG }} bash diff --git a/product/pro/docker-compose.test.yml b/product/pro/docker-compose.test.yml deleted file mode 100644 index e34b86fa5..000000000 --- a/product/pro/docker-compose.test.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - privileged: true - environment: - # uses .env by default - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - TINI_VERSION - - QUARTO_VERSION - - DRIVERS_VERSION - - DRIVERS_VERSION_RHEL - - OS - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/r-session-complete/Justfile b/r-session-complete/Justfile deleted file mode 100755 index da24d8304..000000000 --- a/r-session-complete/Justfile +++ /dev/null @@ -1,87 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "" -PRODUCT := "r-session-complete" -IMAGE_OS := "ubuntu2204" - -RSW_VERSION := "2023.12.1+402.pro1" -RSW_LICENSE := "" - -DRIVERS_VERSION := "2024.03.0" -DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.14" -PYTHON_VERSION_ALT := "3.8.15" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" - -# Build r-session-complete image - just build ubuntu2204 2022.07.2+576.pro12 rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -build OS=IMAGE_OS VERSION=RSW_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - if [[ "{{ OS }}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ - --build-arg DRIVERS_VERSION="${_DRIVERS_VERSION}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test r-session-complete image - just test rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test r-session-complete image interactively - just test-i rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run r-session-complete - just run rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSW_LICENSE }}" ]; then - echo "Please set RSW_LICENSE to a valid RStudio Workbench license before running." - exit 1 - fi - docker run -it --privileged \ - -p 8788:8788 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile deleted file mode 100644 index ce026a86f..000000000 --- a/workbench-for-google-cloud-workstations/Justfile +++ /dev/null @@ -1,116 +0,0 @@ -set dotenv-load -set positional-arguments - -BUILDX_PATH := "" - -PRODUCT := "workbench" -IMAGE_OS := "ubuntu2004" - -RSW_LICENSE := "" -RSW_LICENSE_SERVER := "" - -_make-default-tag: - echo "${IMAGE_REGISTRY_NAME}:$(just -f ../Justfile _get-tag-safe-version "${RSW_VERSION}")" - -get-version: - echo "${RSW_VERSION}" - -get-build-args: - #!/usr/bin/env bash - printf "RSW_VERSION=${RSW_VERSION} - R_VERSION=${R_VERSION} - R_VERSION_ALT=${R_VERSION_ALT} - PYTHON_VERSION=${PYTHON_VERSION} - PYTHON_VERSION_ALT=${PYTHON_VERSION_ALT} - PYTHON_VERSION_JUPYTER=${PYTHON_VERSION_JUPYTER} - QUARTO_VERSION=${QUARTO_VERSION} - DRIVERS_VERSION=${DRIVERS_VERSION} - SRC_IMAGE_NAME=${SRC_IMAGE_NAME} - RSW_DOWNLOAD_URL=${RSW_DOWNLOAD_URL}" - -get-build-tags: - #!/usr/bin/env bash - set -eu - regions=("us-central1" "asia" "europe" "us") - tag_array=() - for region in "${regions[@]}"; do - tag_array+=("$region-docker.pkg.dev/posit-images/cloud-workstations/workbench:${RSW_TAG_VERSION}") - tag_array+=("$region-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest") - done - IFS="," - echo "${tag_array[*]}" - -# Build Workbench image - just build 2022.07.2+576.pro12 rstudio/rstudio-workbench-gcw:2022.07.2 -build *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - CACHE_PATH=${GITHUB_WORKSPACE:-/tmp} - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=${CACHE_PATH}/.buildx-cache --cache-to=type=local,dest=${CACHE_PATH}/.buildx-cache-new,mode=max" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag)) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in ${raw_tag_array[@]}; - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="${RSW_VERSION}" \ - --build-arg R_VERSION="${R_VERSION}" \ - --build-arg R_VERSION_ALT="${R_VERSION_ALT}" \ - --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ - --build-arg PYTHON_VERSION_ALT="${PYTHON_VERSION_ALT}" \ - --build-arg PYTHON_VERSION_JUPYTER="${PYTHON_VERSION_JUPYTER}" \ - --build-arg QUARTO_VERSION="${QUARTO_VERSION}" \ - --build-arg DRIVERS_VERSION="${DRIVERS_VERSION}" \ - --build-arg RSW_DOWNLOAD_URL="${RSW_DOWNLOAD_URL}" \ - --file=./Dockerfile.ubuntu2004 . - - echo ${raw_tag_array[@]} - -# Test Workbench image - just test rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - RSW_VERSION_CLEAN=$(sed "s/daily-/daily+/" <<<"${RSW_VERSION}") - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="${RSW_VERSION_CLEAN}" \ - RSW_LICENSE="{{ RSW_LICENSE }}" \ - RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - DRIVERS_VERSION="${DRIVERS_VERSION}" \ - R_VERSION="${R_VERSION}" \ - R_VERSION_ALT="${R_VERSION_ALT}" \ - PYTHON_VERSION="${PYTHON_VERSION}" \ - PYTHON_VERSION_ALT="${PYTHON_VERSION_ALT}" \ - PYTHON_VERSION_JUPYTER="${PYTHON_VERSION_JUPYTER}" \ - QUARTO_VERSION="${QUARTO_VERSION}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag`: - just test {{ TAG }} bash - -# Run Workbench - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu1804-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euo pipefail - if [ -z "{{ RSW_LICENSE }}" ] && [ -z "{{ RSW_LICENSE_SERVER }}" ]; then - echo "Please set RSW_LICENSE or RSW_LICENSE_SERVER before running." - exit 1 - fi - - docker run -it --privileged \ - ${volume_opts[@]} \ - -p 8787:80 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - -e RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench-for-google-cloud-workstations/docker-compose.test.yml b/workbench-for-google-cloud-workstations/docker-compose.test.yml deleted file mode 100644 index 8de098c6d..000000000 --- a/workbench-for-google-cloud-workstations/docker-compose.test.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSW_VERSION - - R_VERSION - - R_VERSION_ALT - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - PYTHON_VERSION_JUPYTER - - QUARTO_VERSION - - DRIVERS_VERSION - - RSW_LICENSE - - RSW_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" - - "./deps/py_packages.txt:/tmp/py_packages.txt" - - "./deps/r_packages.txt:/tmp/r_packages.txt" diff --git a/workbench-for-microsoft-azure-ml/Justfile b/workbench-for-microsoft-azure-ml/Justfile deleted file mode 100644 index cd9a2fa08..000000000 --- a/workbench-for-microsoft-azure-ml/Justfile +++ /dev/null @@ -1,78 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "workbench" -IMAGE_OS := "ubuntu2204" - -RSW_VERSION := "2023.12.1+402.pro1" -RSW_LICENSE := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -_make-default-tag OS=IMAGE_OS: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" - -# Build Workbench for Azure ML image - just build ubuntu2204 2022.07.2+576.pro12 rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 -build OS=IMAGE_OS VERSION=RSW_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Workbench for Azure ML image - just test rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Workbench for Azure ML image interactively - just test-i rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Workbench for Azure ML - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSW_LICENSE }}" ]; then - echo "Please set RSW_LICENSE to a valid RStudio Workbench license before running." - exit 1 - fi - docker run -it --privileged \ - -p 8787:8787 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench-for-microsoft-azure-ml/docker-compose.test.yml b/workbench-for-microsoft-azure-ml/docker-compose.test.yml deleted file mode 100644 index 53a1d86d0..000000000 --- a/workbench-for-microsoft-azure-ml/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSW_VERSION - - R_VERSION - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - PYTHON_VERSION_JUPYTER - - R_VERSION_ALT - - RSW_LICENSE - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" - - "./test/goss_vars.yaml:/tmp/goss_vars.yaml" diff --git a/workbench/Justfile b/workbench/Justfile deleted file mode 100644 index 785e0ad75..000000000 --- a/workbench/Justfile +++ /dev/null @@ -1,100 +0,0 @@ -set positional-arguments - -BUILDX_PATH := "" - -IMAGE_PREFIX := "rstudio-" -PRODUCT := "workbench" -IMAGE_OS := "ubuntu2204" - -RSW_VERSION := "2023.12.1+402.pro1" -RSW_RELEASE_TYPE := "release" -RSW_LICENSE := "" -RSW_LICENSE_SERVER := "" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.9.17" -PYTHON_VERSION_ALT := "3.8.17" - -PERSIST_LICENSE := "false" -PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") - -_make-default-tag OS=IMAGE_OS VERSION=RSW_VERSION: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{VERSION}})" - -# Build Workbench image - just build ubuntu2204 2022.07.2+576.pro12 rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 -build OS=IMAGE_OS VERSION=RSW_VERSION *TAGS="": - #!/usr/bin/env bash - set -euxo pipefail - BUILDX_ARGS="" - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag {{OS}} {{VERSION}})) - else - raw_tag_array=("{{TAGS}}") - fi - - tag_array=() - for tag in $raw_tag_array - do - tag_array+=("-t" $tag) - done - - docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ - ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg RSW_DOWNLOAD_URL="$(just -f ../ci.Justfile _get-rsw-download-url {{RSW_RELEASE_TYPE}} {{OS}})" \ - --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) . - -# Test Workbench image - just test rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ - RSW_LICENSE="{{ RSW_LICENSE }}" \ - RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} - -# Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu2204-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash - -# Run Workbench - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu2204-2022.07.2-576.pro12 -run TAG=`just _make-default-tag` CMD="": - #!/usr/bin/env bash - set -euxo pipefail - if [ -z "{{ RSW_LICENSE }}" ] && [ -z "{{ RSW_LICENSE_SERVER }}" ]; then - echo "Please set RSW_LICENSE or RSW_LICENSE_SERVER before running." - exit 1 - fi - - volume_opts=() - if [ {{ PERSIST_LICENSE }} = "true" ]; then - if [ {{RSW_LICENSE}} ]; then - echo "Volumes will be configured to persist license state data for an activation key." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes key {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - elif [ {{RSW_LICENSE_SERVER}} ]; then - echo "Volumes will be configured to persist license state data for a floating license server." - volume_opts=$(just -f ../Justfile _config-license-persist-volumes float {{PRODUCT}} {{PERSIST_LICENSE_DIR}}) - fi - fi - - docker run -it --privileged \ - ${volume_opts[@]} \ - -p 8787:8787 \ - -e RSW_LICENSE="{{ RSW_LICENSE }}" \ - -e RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - "{{ TAG }}" {{ CMD }} diff --git a/workbench/docker-compose.test.yml b/workbench/docker-compose.test.yml deleted file mode 100644 index db54a879d..000000000 --- a/workbench/docker-compose.test.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '2.3' -services: - - sut: - image: $IMAGE_NAME - command: /run_tests.sh - entrypoint: [] - environment: - # uses .env by default - - RSW_VERSION - - R_VERSION - - PYTHON_VERSION - - PYTHON_VERSION_ALT - - PYTHON_VERSION_JUPYTER - - R_VERSION_ALT - - RSW_LICENSE - - RSW_LICENSE_SERVER - volumes: - - "./test/run_tests.sh:/run_tests.sh" - - "./test/goss.yaml:/tmp/goss.yaml" From 0dbf4a53e28ef2f56ba4e072375e1a6c85e8317f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 11:46:02 -0700 Subject: [PATCH 13/37] Increase timeout for checking Jupyter --- r-session-complete/test/goss.yaml | 1 + workbench-for-google-cloud-workstations/test/goss.yaml | 1 + workbench-for-microsoft-azure-ml/test/goss.yaml | 1 + workbench/test/goss.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/r-session-complete/test/goss.yaml b/r-session-complete/test/goss.yaml index 94637931e..72cc49e58 100644 --- a/r-session-complete/test/goss.yaml +++ b/r-session-complete/test/goss.yaml @@ -22,6 +22,7 @@ file: command: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/{{.Env.PYTHON_VERSION}}/bin/jupyter nbconvert --to notebook --stdin --stdout": title: jupyter_works + timeout: 60000 exit-status: 0 # Ensure correct R version diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index e2d692fad..833eb139a 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -177,6 +177,7 @@ command: ] Test Jupyter works: exec: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout" + timeout: 60000 exit-status: 0 Check primary Python is version {{.Env.PYTHON_VERSION}}: exec: /opt/python/{{.Env.PYTHON_VERSION}}/bin/python --version diff --git a/workbench-for-microsoft-azure-ml/test/goss.yaml b/workbench-for-microsoft-azure-ml/test/goss.yaml index 05269e7d9..2c8ba3fea 100644 --- a/workbench-for-microsoft-azure-ml/test/goss.yaml +++ b/workbench-for-microsoft-azure-ml/test/goss.yaml @@ -101,6 +101,7 @@ command: ] "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": title: jupyter_works + timeout: 60000 exit-status: 0 # Ensure correct python version "/opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version": diff --git a/workbench/test/goss.yaml b/workbench/test/goss.yaml index b48143786..21379f9e7 100644 --- a/workbench/test/goss.yaml +++ b/workbench/test/goss.yaml @@ -113,6 +113,7 @@ command: ] "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": title: jupyter_works + timeout: 60000 exit-status: 0 # Ensure correct python version "/opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version": From 9bd2b71130b58c20ce0c978cde5678544ff34e8d Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 11:46:16 -0700 Subject: [PATCH 14/37] Convert content builds to use bake --- .github/workflows/build-bake.yaml | 20 ++++++++ content/docker-bake.hcl | 82 +++++++++++++++++++++++++++++++ content/matrix.json | 23 --------- content/pro/Dockerfile.ubuntu1804 | 5 +- content/pro/Dockerfile.ubuntu2204 | 5 +- 5 files changed, 104 insertions(+), 31 deletions(-) create mode 100644 content/docker-bake.hcl delete mode 100644 content/matrix.json diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 25c396d94..878bd2aef 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -42,3 +42,23 @@ jobs: targets: "build" push: true + build-content: + runs-on: ubuntu-latest-4x + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/bake-action@v4 + with: + push: false + + - name: Build and push + if: github.ref == 'refs/heads/main' + uses: docker/bake-action@v4 + with: + push: true + diff --git a/content/docker-bake.hcl b/content/docker-bake.hcl new file mode 100644 index 000000000..d277fada1 --- /dev/null +++ b/content/docker-bake.hcl @@ -0,0 +1,82 @@ +variable CONTENT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu1804", os_alt = "bionic", r = "3.1.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.2.5", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.3.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.4.4", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.5.3", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "2.7.18", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "3.6.3", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.6.13", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.7.10", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.0.5", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.8.8", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.0", py = "3.9.2", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu1804", os_alt = "bionic", r = "4.1.3", py = "3.10.4", drivers = "2024.03.0", quarto = "1.0.37"}, + {os = "ubuntu2204", os_alt = "jammy", r = "3.6.3", py = "3.8.16", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.0.5", py = "3.9.16", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.1.3", py = "3.10.11", drivers = "2024.03.0", quarto = "1.3.340"}, + {os = "ubuntu2204", os_alt = "jammy", r = "4.2.2", py = "3.11.3", drivers = "2024.03.0", quarto = "1.3.340"}, + ] + } +} + +group "default" { + targets = ["base", "pro"] +} + +target "base" { + name = "content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + + tags = [ + "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", + "ghcr.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", + "docker.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os}", + "docker.io.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", + ] + output = [ + "type=cacheonly", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "base" + + matrix = CONTENT_BUILD_MATRIX + args = { + R_VERSION = "${builds.r}" + PYTHON_VERSION = "${builds.py}" + QUARTO_VERSION = "${builds.quarto}" + } +} + +target "pro" { + name = "content-pro-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + + tags = [ + "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", + "ghcr.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", + "docker.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os}", + "docker.io.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", + ] + output = [ + "type=cacheonly", + ] + + contexts = { + content-base = "target:content-base-r${replace(builds.r, ".", "-")}-py${replace(builds.py, ".", "-")}-${builds.os}" + } + + dockerfile = "Dockerfile.${builds.os}" + context = "pro" + + matrix = CONTENT_BUILD_MATRIX + args = { + R_VERSION = "${builds.r}" + DRIVERS_VERSION = "${builds.drivers}" + } +} diff --git a/content/matrix.json b/content/matrix.json deleted file mode 100644 index 163b8ddd0..000000000 --- a/content/matrix.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - {"r": "3.1.3", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.2.5", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.3.3", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.4.4", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.4.4", "py": "3.7.10", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.5.3", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.5.3", "py": "3.7.10", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "2.7.18", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "3.8.8", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.6.13", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.7.10", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.8.8", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.0.5", "py": "3.9.2", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.1.0", "py": "3.8.8", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.1.0", "py": "3.9.2", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "4.1.3", "py": "3.10.4", "drivers": "2024.03.0", "os": "ubuntu1804", "os_alt": "bionic"}, - {"r": "3.6.3", "py": "3.8.16", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"}, - {"r": "4.0.5", "py": "3.9.16", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"}, - {"r": "4.1.3", "py": "3.10.11", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"}, - {"r": "4.2.2", "py": "3.11.3", "drivers": "2024.03.0", "os": "ubuntu2204", "os_alt": "jammy"} -] diff --git a/content/pro/Dockerfile.ubuntu1804 b/content/pro/Dockerfile.ubuntu1804 index e3be3d7fd..1efd4901c 100644 --- a/content/pro/Dockerfile.ubuntu1804 +++ b/content/pro/Dockerfile.ubuntu1804 @@ -1,7 +1,4 @@ -ARG PYTHON_VERSION -ARG R_VERSION -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/content-base:r${R_VERSION}-py${PYTHON_VERSION}-ubuntu1804 +FROM content-base LABEL maintainer="RStudio Docker " diff --git a/content/pro/Dockerfile.ubuntu2204 b/content/pro/Dockerfile.ubuntu2204 index 4a8066b54..1efd4901c 100644 --- a/content/pro/Dockerfile.ubuntu2204 +++ b/content/pro/Dockerfile.ubuntu2204 @@ -1,7 +1,4 @@ -ARG PYTHON_VERSION -ARG R_VERSION -ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/content-base:r${R_VERSION}-py${PYTHON_VERSION}-ubuntu2204 +FROM content-base LABEL maintainer="RStudio Docker " From ca6e41f7fd9a4fb7b2d4b52c237ac9ada47ce2ff Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 11:55:18 -0700 Subject: [PATCH 15/37] Target content docker-bake file --- .github/workflows/build-bake.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 878bd2aef..314edb80f 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -54,11 +54,13 @@ jobs: - name: Build and push uses: docker/bake-action@v4 with: + workdir: content/. push: false - name: Build and push if: github.ref == 'refs/heads/main' uses: docker/bake-action@v4 with: + workdir: content/. push: true From 6ae5bb245530b23accfc9ae6f9cfe477ffee62fc Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 13:08:05 -0700 Subject: [PATCH 16/37] Print metadata after build --- .github/workflows/build-bake.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 314edb80f..c31876d47 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -25,7 +25,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push + - name: Build and test + id: build uses: docker/bake-action@v4 with: targets: "build-test" @@ -35,13 +36,17 @@ jobs: run: | just -f build.justfile test-connect - - name: Build and push + - name: Push + id: push if: github.ref == 'refs/heads/main' uses: docker/bake-action@v4 with: targets: "build" push: true + - name: Print Metadata + run: echo "${{ steps.push.outputs.metadata }}" + build-content: runs-on: ubuntu-latest-4x steps: @@ -51,16 +56,20 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push + - name: Build + id: build uses: docker/bake-action@v4 with: workdir: content/. push: false - - name: Build and push + - name: Push + id: push if: github.ref == 'refs/heads/main' uses: docker/bake-action@v4 with: workdir: content/. push: true + - name: Print Metadata + run: echo "${{ steps.push.outputs.metadata }}" From 0a1faf2b64793fa0ace27e845fd280e6ab059a98 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 14:02:09 -0700 Subject: [PATCH 17/37] Enable GHA caching --- .github/workflows/build-bake.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index c31876d47..1038e3717 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -9,6 +9,11 @@ name: Bake Docker Images jobs: build: runs-on: ubuntu-latest-4x + + concurrency: + group: bake-${{ github.ref }} + cancel-in-progress: true + steps: - name: Checkout uses: actions/checkout@v4 @@ -29,6 +34,9 @@ jobs: id: build uses: docker/bake-action@v4 with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd targets: "build-test" push: false @@ -41,6 +49,9 @@ jobs: if: github.ref == 'refs/heads/main' uses: docker/bake-action@v4 with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd targets: "build" push: true @@ -49,6 +60,11 @@ jobs: build-content: runs-on: ubuntu-latest-4x + + concurrency: + group: bake-content-${{ github.ref }} + cancel-in-progress: true + steps: - name: Checkout uses: actions/checkout@v4 @@ -60,6 +76,9 @@ jobs: id: build uses: docker/bake-action@v4 with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd workdir: content/. push: false @@ -68,6 +87,9 @@ jobs: if: github.ref == 'refs/heads/main' uses: docker/bake-action@v4 with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd workdir: content/. push: true From a7b55d322d13d8581f2708f561b072f0d6aadb19 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 14:06:30 -0700 Subject: [PATCH 18/37] Default to image output --- content/docker-bake.hcl | 4 ++-- docker-bake.hcl | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/content/docker-bake.hcl b/content/docker-bake.hcl index d277fada1..3205ca24d 100644 --- a/content/docker-bake.hcl +++ b/content/docker-bake.hcl @@ -40,7 +40,7 @@ target "base" { "docker.io.io/rstudio/content-base:r${builds.r}-py${builds.py}-${builds.os_alt}", ] output = [ - "type=cacheonly", + "type=image", ] dockerfile = "Dockerfile.${builds.os}" @@ -64,7 +64,7 @@ target "pro" { "docker.io.io/rstudio/content-pro:r${builds.r}-py${builds.py}-${builds.os_alt}", ] output = [ - "type=cacheonly", + "type=image", ] contexts = { diff --git a/docker-bake.hcl b/docker-bake.hcl index 403544935..7a3a15c12 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -221,7 +221,7 @@ target "base" { labels = { "maintainer" = "Posit Docker " } - output = ["type=cacheonly"] + output = ["type=image"] } target "product-base" { @@ -233,9 +233,6 @@ target "product-base" { "ghcr.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", "docker.io/rstudio/product-base:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", ] - output = [ - "type=cacheonly", - ] dockerfile = "Dockerfile.${builds.os}" context = "product/base" @@ -274,9 +271,6 @@ target "product-base-pro" { "ghcr.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", "docker.io/rstudio/product-base-pro:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", ] - output = [ - "type=cacheonly", - ] dockerfile = "Dockerfile.${builds.os}" context = "product/pro" @@ -363,7 +357,7 @@ target "connect" { ] # We output Connect to OCI so it can be pulled in for testing later on. output = [ - "type=cacheonly", + "type=image", "type=oci,tar=false,dest=./.out/connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" ] From 32a90d6731e234d38e91d57745be364bd17c8477 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 14:07:01 -0700 Subject: [PATCH 19/37] Remove debug line --- .github/workflows/build-bake.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 1038e3717..f58f42791 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -55,9 +55,6 @@ jobs: targets: "build" push: true - - name: Print Metadata - run: echo "${{ steps.push.outputs.metadata }}" - build-content: runs-on: ubuntu-latest-4x @@ -92,6 +89,3 @@ jobs: *.cache-to=type=gha,compression=zstd workdir: content/. push: true - - - name: Print Metadata - run: echo "${{ steps.push.outputs.metadata }}" From 9e9ac3a9c85e9c54c618377a04fedccf22b493b9 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 5 Apr 2024 14:08:31 -0700 Subject: [PATCH 20/37] Suppress DL3006 --- hadolint.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadolint.yaml b/hadolint.yaml index bf071e06c..6548cc9c6 100644 --- a/hadolint.yaml +++ b/hadolint.yaml @@ -1,5 +1,7 @@ failure-threshold: warning ignored: + # Suppresses warning for not explicitly tagging FROM images. Docker buildx bake is handling FROM image injection. + - DL3006 # Suppresses warning for version pinning on apt installs. We currently do not practice this approach. - DL3008 # Suppresses warning for version pinning on yum installs. We currently do not practice this approach. From 929f9bc77789782cfe8c6c077a5830a2dc10a0a1 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 06:54:58 -0700 Subject: [PATCH 21/37] Extend timeouts for GCW --- docker-bake.hcl | 2 +- workbench-for-google-cloud-workstations/test/goss.yaml | 5 +++++ workbench-for-google-cloud-workstations/test/run_tests.sh | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 7a3a15c12..a49f8f3b8 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -83,7 +83,7 @@ variable WORKBENCH_BUILD_MATRIX { variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { default = { builds = [ - {os = "ubuntu2004", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, + {os = "ubuntu2004", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.7", py_alternate = "3.10.13"}, ] } } diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index 833eb139a..9209c995c 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -181,18 +181,21 @@ command: exit-status: 0 Check primary Python is version {{.Env.PYTHON_VERSION}}: exec: /opt/python/{{.Env.PYTHON_VERSION}}/bin/python --version + timeout: 60000 exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION}}" ] Check alternate Python is version {{.Env.PYTHON_VERSION_ALT}}: exec: /opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version + timeout: 60000 exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION_ALT}}" ] Check Jupyter venv uses Python {{.Env.PYTHON_VERSION_JUPYTER}}: exec: /opt/python/jupyter/bin/python --version + timeout: 60000 exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION_JUPYTER}}" @@ -209,9 +212,11 @@ command: {{- range $py_package_list }} Check Python {{ $python_version }} has "{{.}}" installed: exec: /opt/python/{{$python_version}}/bin/pip show {{.}} + timeout: 60000 exit-status: 0 Check Python {{ $python_version_alt }} has "{{.}}" installed: exec: /opt/python/{{$python_version_alt}}/bin/pip show {{.}} + timeout: 60000 exit-status: 0 {{end}} {{ $r_version := .Env.R_VERSION }} diff --git a/workbench-for-google-cloud-workstations/test/run_tests.sh b/workbench-for-google-cloud-workstations/test/run_tests.sh index dd9d1b236..10d390043 100755 --- a/workbench-for-google-cloud-workstations/test/run_tests.sh +++ b/workbench-for-google-cloud-workstations/test/run_tests.sh @@ -20,7 +20,7 @@ echo "--> Startup complete" GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} GOSS_VERSION=${GOSS_VERSION:-0.4.6} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5} # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ From 165eb819f5b1fe58c9eb153051bad6a8a5c7861b Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 07:09:36 -0700 Subject: [PATCH 22/37] Add timeout to r-session-complete jupyter --- r-session-complete/test/goss.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/r-session-complete/test/goss.yaml b/r-session-complete/test/goss.yaml index 72cc49e58..09760e7c0 100644 --- a/r-session-complete/test/goss.yaml +++ b/r-session-complete/test/goss.yaml @@ -50,4 +50,5 @@ command: "jupyter --version": title: jupyter_in_path_var + timeout: 60000 exit-status: 0 From e54f5647b5028eb2c80f144ca9529a9c3fcc922d Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 07:16:09 -0700 Subject: [PATCH 23/37] Cleanup justfiles --- Justfile | 28 +++++++------------- ci.Justfile | 66 ---------------------------------------------- docker-bake.hcl | 6 ++++- docker-compose.yml | 2 +- 4 files changed, 15 insertions(+), 87 deletions(-) diff --git a/Justfile b/Justfile index 591527882..3b2510765 100644 --- a/Justfile +++ b/Justfile @@ -112,13 +112,11 @@ update-rsw-versions: sed {{ sed_vars }} "s/RSW_VERSION:.*/RSW_VERSION: {{ RSW_VERSION }}/g" docker-compose.yml sed {{ sed_vars }} "s/rstudio\/rstudio-workbench:.*/rstudio\/rstudio-workbench:$(just _get-clean-version {{ RSW_VERSION }})/g" docker-compose.yml sed {{ sed_vars }} "s/^RSW_VERSION := .*/RSW_VERSION := \"{{ RSW_VERSION }}\"/g" \ - r-session-complete/Justfile \ - workbench/Justfile \ - workbench-for-microsoft-azure-ml/Justfile \ Justfile sed {{ sed_vars }} "s/[0-9]\{4\}\.[0-9]\{1,2\}\.[0-9]\{1,2\}/`just _get-clean-version {{ RSW_VERSION }}`/g" \ workbench/README.md \ r-session-complete/README.md + sed -i '/variable WORKBENCH_VERSION/!b;n;c\ \ \ \ default = "{{ RSW_VERSION }}"' docker-bake.hcl # just RSPM_VERSION=1.2.3 update-rspm-versions update-rspm-versions: @@ -133,6 +131,7 @@ update-rspm-versions: package-manager/Justfile \ Justfile sed {{ sed_vars }} -E "s/[0-9]{4}\.[0-9]{1,2}\.[0-9]{1,2}/`just _get-clean-version {{ RSPM_VERSION }}`/g" package-manager/README.md + sed -i '/variable PACKAGE_MANAGER_VERSION/!b;n;c\ \ \ \ default = "{{ RSPM_VERSION }}"' docker-bake.hcl # just RSC_VERSION=1.2.3 update-rsc-versions update-rsc-versions: @@ -151,9 +150,11 @@ update-rsc-versions: sed {{ sed_vars }} -E "s/[0-9]{4}\.[0-9]{1,2}\.[0-9]{1,2}/`just _get-clean-version {{ RSC_VERSION }}`/g" \ connect/README.md \ connect-content-init/README.md + sed -i '/variable CONNECT_VERSION/!b;n;c\ \ \ \ default = "{{ RSC_VERSION }}"' docker-bake.hcl # just R_VERSION=3.2.1 R_VERSION_ALT=4.1.0 update-r-versions -update-r-versions: +update-r-versions: update-default-r-versions +update-default-r-versions: #!/usr/bin/env bash set -euxo pipefail # Update primary R versions @@ -197,7 +198,8 @@ update-r-versions: ci.Justfile # just PYTHON_VERSION=3.9.5 PYTHON_VERSION_ALT=3.8.10 update-py-versions -update-py-versions: +update-py-versions: update-default-py-versions +update-default-py-versions: #!/usr/bin/env bash set -euxo pipefail # Update primary Python versions @@ -267,6 +269,7 @@ update-drivers-versions: r-session-complete/Justfile \ product/pro/Justfile \ ci.Justfile + sed -i '/variable DRIVERS_VERSION/!b;n;c\ \ \ \ default = "{{ RSC_VERSION }}"' docker-bake.hcl update-quarto-versions: #!/usr/bin/env bash @@ -283,20 +286,7 @@ update-quarto-versions: connect/rstudio-connect.gcfg sed {{ sed_vars }} "s/qver=\${QUARTO_VERSION:-.*}/qver=\${QUARTO_VERSION:-{{ QUARTO_VERSION }}}/g" \ content/base/maybe_install_quarto.sh - - -# just test-image preview workbench 12.0.11-8 tag1 tag2 tag3 ... -test-image $PRODUCT $VERSION +IMAGES: - #!/usr/bin/env bash - set -euxo pipefail - IMAGES="{{IMAGES}}" - read -ra IMAGE_ARRAY <<<"$IMAGES" - just \ - R_VERSION={{R_VERSION}} \ - R_VERSION_ALT={{R_VERSION_ALT}} \ - PYTHON_VERSION={{PYTHON_VERSION}} \ - PYTHON_VERSION_ALT={{PYTHON_VERSION_ALT}} \ - $PRODUCT/test "${IMAGE_ARRAY[0]}" "$VERSION" + sed -i '/variable DEFAULT_QUARTO_VERSION/!b;n;c\ \ \ \ default = "{{ QUARTO_VERSION }}"' docker-bake.hcl # just lint workbench ubuntu2204 lint $PRODUCT $OS: diff --git a/ci.Justfile b/ci.Justfile index 392613192..06324e8ec 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -59,55 +59,6 @@ _get-rsw-download-url TYPE OS: get-version +NARGS: ./tools/get-version.py {{NARGS}} -# just get-base-args ubuntu2204 base|pro -get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: - #!/usr/bin/env bash - set -euxo pipefail - if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then - SRC_IMAGE_NAME="" - CTX_PATH="./product/base" - FILE_PATH="./product/base/Dockerfile.${OS}" - elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then - SRC_IMAGE_NAME="product-base" - CTX_PATH="./product/pro" - FILE_PATH="./product/pro/Dockerfile.${OS}" - fi - if [[ $BRANCH != "main" ]]; then - SRC_IMAGE_NAME="${SRC_IMAGE_NAME}-dev" - fi - - if [[ "${OS}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - printf "R_VERSION={{ R_VERSION }} - R_VERSION_ALT={{ R_VERSION_ALT }} - PYTHON_VERSION={{ PYTHON_VERSION }} - PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} - QUARTO_VERSION={{ QUARTO_VERSION }} - DRIVERS_VERSION=${_DRIVERS_VERSION} - SRC_IMAGE_NAME=${SRC_IMAGE_NAME}" - -# just get-base-tags ubuntu2204 base|pro -get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: - #!/usr/bin/env bash - set -euxo pipefail - IMAGE_NAME="" - if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then - IMAGE_NAME="product-base" - elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then - IMAGE_NAME="product-base-pro" - fi - if [[ $BRANCH != "main" ]]; then - IMAGE_NAME="${IMAGE_NAME}-dev" - fi - - echo ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}},\ - ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}},\ - ghcr.io/rstudio/${IMAGE_NAME}:${OS} - # just get-product-args connect ubuntu2204 2023.05.0 get-product-args $PRODUCT $OS $VERSION $USE_S3="false" $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`: #!/usr/bin/env bash @@ -286,20 +237,3 @@ get-prerelease-tags $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: done tags=$(IFS="," ; echo "${tag_array[*]}") echo "${tags}" - -# just get-content-args 4.2.3 3.9.17 -get-content-args r-ver py-ver drivers-ver="": - #!/usr/bin/env bash - printf "R_VERSION={{r-ver}} - PYTHON_VERSION={{py-ver}} - DRIVERS_VERSION={{drivers-ver}}" - -# just get-content-tags content-base|content-pro 4.2.3 3.9.17 ubuntu2204 -get-content-tags image-name r-ver py-ver os: - #!/usr/bin/env bash - OS=$(just _parse-os {{os}}) - OS_ALT=$(just _rev-parse-os {{os}}) - echo rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS},\ - ghcr.io/rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS},\ - rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS_ALT},\ - ghcr.io/rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS_ALT} diff --git a/docker-bake.hcl b/docker-bake.hcl index a49f8f3b8..1e5e4545d 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -15,6 +15,10 @@ variable DRIVERS_VERSION { default = "2024.03.0" } +variable DEFAULT_QUARTO_VERSION { + default = "1.4.553" +} + function get_drivers_version { params = [os] result = os == "centos7" ? "${DRIVERS_VERSION}-1" : DRIVERS_VERSION @@ -534,7 +538,7 @@ target "workbench-for-google-cloud-workstations" { PYTHON_VERSION_ALT = builds.py_alternate PYTHON_VERSION_JUPYTER = builds.py_alternate JUPYTERLAB_VERSION = "3.6.5" - QUARTO_VERSION = "1.4.550" + QUARTO_VERSION = DEFAULT_QUARTO_VERSION DRIVERS_VERSION = get_drivers_version(builds.os) RSW_VERSION = WORKBENCH_VERSION RSW_NAME = "rstudio-workbench" diff --git a/docker-compose.yml b/docker-compose.yml index 149b95df2..88ef5b309 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: dockerfile: "Dockerfile.${IMAGE_OS:-ubuntu2204}" args: RSW_VERSION: 2023.12.1+402.pro1 - image: rstudio/rstudio-workbench:2023.12.0 + image: rstudio/rstudio-workbench:2023.12.1 environment: RSW_LICENSE: ${RSW_LICENSE} LICENSE_SERVER: ${RSW_LICENSE_SERVER} From 68e44ba65256b4c9baf145123109d7b3134b5e32 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 10:23:46 -0700 Subject: [PATCH 24/37] Update tagging for bake release builds --- .github/workflows/build-bake.yaml | 13 +-- NEWS.md | 4 + docker-bake.hcl | 147 ++++++++++++++++++------------ 3 files changed, 98 insertions(+), 66 deletions(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index f58f42791..5334f0834 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -1,13 +1,14 @@ on: - push: - branches: - - main - - dev - pull_request: + push: + branches: + - main + - dev + pull_request: -name: Bake Docker Images +name: Release - Build, Test, and Push jobs: build: + name: Build and Test images runs-on: ubuntu-latest-4x concurrency: diff --git a/NEWS.md b/NEWS.md index 2a981260b..2265944d3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,10 @@ changed in each image. This file only captures pervasive, repository-wide changes. +# 2024-04-08 + +- Change build orchestration to use `docker buildx bake` for all images. + # 2024-03-14 - Update Professional Drivers to 2024.03.0 diff --git a/docker-bake.hcl b/docker-bake.hcl index 1e5e4545d..90e6aa70f 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -19,11 +19,51 @@ variable DEFAULT_QUARTO_VERSION { default = "1.4.553" } +function workbench_version_clean { + params = [] + result = split("+", WORKBENCH_VERSION)[0] +} + +function get_os_alt_name { + params = [os] + result = os == "ubuntu2204" ? "jammy" : os +} + function get_drivers_version { params = [os] result = os == "centos7" ? "${DRIVERS_VERSION}-1" : DRIVERS_VERSION } +function get_centos_tags { + params = [os, product, product_version] + result = [ + "ghcr.io/rstudio/${product}:${os}", + "ghcr.io/rstudio/${product}:${os}-${product_version}", + "docker.io/rstudio/${product}:${os}", + "docker.io/rstudio/${product}:${os}-${product_version}", + ] +} + +function get_ubuntu_tags { + params = [os, product, product_version] + result = [ + "ghcr.io/rstudio/${product}:${os}", + "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}", + "ghcr.io/rstudio/${product}:${os}-${product_version}", + "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}-${product_version}", + "docker.io/rstudio/${product}:${os}", + "docker.io/rstudio/${product}:${get_os_alt_name(os)}", + "docker.io/rstudio/${product}:${os}-${product_version}", + "docker.io/rstudio/${product}:${get_os_alt_name(os)}-${product_version}", + ] +} + +# FIXME: There's an obnoxious amount of hardcoding here due to restrictions with what bake can actually interpret from HCL. +function get_tags { + params = [os, product, product_version] + result = os == "ubuntu2204" ? get_ubuntu_tags(os, product, product_version) : get_centos_tags(os, product, product_version) +} + ### Build matrices ### variable BASE_BUILD_MATRIX { @@ -313,11 +353,8 @@ target "package-manager" { inherits = ["base"] target = "build" - name = "package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - tags = [ - "ghcr.io/rstudio/rstudio-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/rstudio-package-manager:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + name = "package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" + tags = get_tags(builds.os, "package-manager", PACKAGE_MANAGER_VERSION) dockerfile = "Dockerfile.${builds.os}" context = "package-manager" @@ -336,14 +373,14 @@ target "package-manager" { } target "test-package-manager" { - inherits = ["package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}"] target = "test" - name = "test-package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + name = "test-package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" tags = [] contexts = { - build = "target:package-manager-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + build = "target:package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" } matrix = PACKAGE_MANAGER_BUILD_MATRIX @@ -354,15 +391,13 @@ target "connect" { inherits = ["base"] target = "build" - name = "connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - tags = [ - "ghcr.io/rstudio/rstudio-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/rstudio-connect:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + name = "connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect", CONNECT_VERSION) + # We output Connect to OCI so it can be pulled in for testing later on. output = [ "type=image", - "type=oci,tar=false,dest=./.out/connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + "type=oci,tar=false,dest=./.out/connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" ] dockerfile = "Dockerfile.${builds.os}" @@ -383,14 +418,14 @@ target "connect" { # FIXME: This target requires a privileged environment which bake cannot provide target "test-connect" { - inherits = ["connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}"] target = "test" - name = "test-connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + name = "test-connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" tags = [] contexts = { - build = "target:connect-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + build = "target:connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" } matrix = CONNECT_BUILD_MATRIX @@ -401,12 +436,7 @@ target "connect-content-init" { target = "build" name = "connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" - tags = [ - "ghcr.io/rstudio/product-connect-content-init:${builds.os}", - "ghcr.io/rstudio/product-connect-content-init:${builds.os}-${CONNECT_VERSION}", - "docker.io/rstudio/product-connect-content-init:${builds.os}", - "docker.io/rstudio/product-connect-content-init:${builds.os}-${CONNECT_VERSION}", - ] + tags = get_tags(builds.os, "rstudio-connect-content-init", CONNECT_VERSION) dockerfile = "Dockerfile.${builds.os}" context = "connect-content-init" @@ -437,11 +467,8 @@ target "r-session-complete" { inherits = ["base"] target = "build" - name = "r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - tags = [ - "ghcr.io/rstudio/product-r-session-complete:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/product-r-session-complete:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + name = "r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = get_tags(builds.os, "r-session-complete", workbench_version_clean()) dockerfile = "Dockerfile.${builds.os}" context = "r-session-complete" @@ -463,14 +490,14 @@ target "r-session-complete" { } target "test-r-session-complete" { - inherits = ["r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] target = "test" - name = "test-r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + name = "test-r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" tags = [] contexts = { - build = "target:r-session-complete-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + build = "target:r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" } matrix = R_SESSION_COMPLETE_BUILD_MATRIX @@ -479,11 +506,8 @@ target "test-r-session-complete" { target "workbench" { inherits = ["base"] - name = "workbench-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - tags = [ - "ghcr.io/rstudio/product-workbench:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/product-workbench:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + name = "workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench", workbench_version_clean()) dockerfile = "Dockerfile.${builds.os}" context = "workbench" @@ -505,13 +529,13 @@ target "workbench" { } target "test-workbench" { - inherits = ["workbench-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] - name = "test-workbench-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + name = "test-workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" tags = [] contexts = { - build = "target:workbench-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + build = "target:workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" } matrix = WORKBENCH_BUILD_MATRIX @@ -521,10 +545,16 @@ target "test-workbench" { target "workbench-for-google-cloud-workstations" { inherits = ["base"] - name = "workbench-for-google-cloud-workstation-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + name = "workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" tags = [ - "ghcr.io/rstudio/workbench-for-google-cloud-workstation:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/workbench-for-google-cloud-workstation:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench:${workbench_version_clean()}", + "us-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "us-docker.pkg.dev/posit-images/cloud-workstations/workbench:${workbench_version_clean()}", + "europe-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "europe-docker.pkg.dev/posit-images/cloud-workstations/workbench:${workbench_version_clean()}", + "asia-docker.pkg.dev/posit-images/cloud-workstations/workbench:latest", + "asia-docker.pkg.dev/posit-images/cloud-workstations/workbench:${workbench_version_clean()}", ] dockerfile = "Dockerfile.${builds.os}" @@ -547,13 +577,13 @@ target "workbench-for-google-cloud-workstations" { } target "test-workbench-for-google-cloud-workstations" { - inherits = ["workbench-for-google-cloud-workstation-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] - name = "test-workbench-for-google-cloud-workstation-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + name = "test-workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" tags = [] contexts = { - build = "target:workbench-for-google-cloud-workstation-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + build = "target:workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" } matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX @@ -565,7 +595,7 @@ target "build-workbench-for-microsoft-azure-ml" { inherits = ["base"] target = "build" - name = "build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + name = "build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" dockerfile = "Dockerfile.${builds.os}" context = "workbench-for-microsoft-azure-ml" @@ -588,44 +618,41 @@ target "build-workbench-for-microsoft-azure-ml" { } target "scan-workbench-for-microsoft-azure-ml" { - inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] target = "clamav" - name = "scan-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + name = "scan-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" contexts = { - build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" } matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX } target "test-workbench-for-microsoft-azure-ml" { - inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] target = "test" - name = "test-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + name = "test-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" contexts = { - build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" } matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX } target "workbench-for-microsoft-azure-ml" { - inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + inherits = ["build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}"] target = "final" - name = "workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - tags = [ - "ghcr.io/rstudio/product-workbench-for-microsoft-azure-ml:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - "docker.io/rstudio/product-workbench-for-microsoft-azure-ml:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", - ] + name = "workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench-for-microsoft-azure-ml", workbench_version_clean()) contexts = { - build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" - clamav = "target:scan-workbench-for-microsoft-azure-ml-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + clamav = "target:scan-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" } matrix = WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX From 7df68cb06aa750af5f8e032d7d71d23701e9d716 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 12:47:08 -0700 Subject: [PATCH 25/37] Implement bake preview builds --- .github/workflows/build-bake-preview.yaml | 83 +++ build.justfile | 63 +++ docker-bake.hcl | 2 +- docker-bake.preview.hcl | 617 ++++++++++++++++++++++ tools/bake_test_command_extract.py | 7 +- 5 files changed, 768 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/build-bake-preview.yaml create mode 100644 docker-bake.preview.hcl diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml new file mode 100644 index 000000000..61dd4c37c --- /dev/null +++ b/.github/workflows/build-bake-preview.yaml @@ -0,0 +1,83 @@ +on: + schedule: + - cron: '0 8 * * *' + - cron: '0 9 * * *' + push: + branches: + - main + - dev + - dev-rspm + pull_request: + +name: Preview - Build, Test, and Push +jobs: + build-preview: + runs-on: ubuntu-latest-4x + + concurrency: + group: bake-preview-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Check Out main Branch + if: github.event.schedule == '0 8 * * *' + uses: actions/checkout@v3 + with: + ref: 'main' + + - name: Check Out Repo at Triggered Branch + if: github.event.schedule != '0 8 * * *' + uses: actions/checkout@v3 + + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Get Version + id: get-version + run: | + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) + echo "WORKBENCH_DAILY_VERSION=$WORKBENCH_DAILY_VERSION" >> $GITHUB_OUTPUT + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) + echo "WORKBENCH_PREVIEW_VERSION=$WORKBENCH_PREVIEW_VERSION" >> $GITHUB_OUTPUT + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) + echo "PACKAGE_MANAGER_DAILY_VERSION=$PACKAGE_MANAGER_DAILY_VERSION" >> $GITHUB_OUTPUT + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) + echo "CONNECT_DAILY_VERSION=$CONNECT_DAILY_VERSION" >> $GITHUB_OUTPUT + + - name: Build and test + id: build + uses: docker/bake-action@v4 + env: + WORKBENCH_DAILY_VERSION: ${{ steps.get-version.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ steps.get-version.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ steps.get-version.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ steps.get-version.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} + with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd + targets: "build-test" + push: false + + - name: Test Connect (privileged container) + run: | + just -f build.justfile test-connect-preview + + - name: Push + id: push + if: github.ref == 'refs/heads/main' + uses: docker/bake-action@v4 + with: + set: | + *.cache-from=type=gha + *.cache-to=type=gha,compression=zstd + targets: "build" + push: true + diff --git a/build.justfile b/build.justfile index 1467f3c48..0ad10bea2 100755 --- a/build.justfile +++ b/build.justfile @@ -20,24 +20,68 @@ bake target: just -f {{justfile()}} create-builder || true docker buildx bake --builder=posit-builder -f docker-bake.hcl {{target}} +preview-bake target branch="$(git branch --show-current)": + just -f {{justfile()}} create-builder || true + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + docker buildx bake --builder=posit-builder -f docker-bake.preview.hcl {{target}} + # just plan plan: docker buildx bake -f docker-bake.hcl --print +preview-plan branch="$(git branch --show-current)": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + docker buildx bake -f docker-bake.preview.hcl --print + # just build build: just -f {{justfile()}} bake build +preview-build branch="$(git branch --show-current)": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + just -f {{justfile()}} bake preview-build + # just test test: just -f {{justfile()}} bake test just -f {{justfile()}} test-connect ubuntu2204 +# just preview-test +preview-test branch="$(git branch --show-current)": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + just -f {{justfile()}} bake preview-test + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + just -f {{justfile()}} test-connect-preview ubuntu2204 + # just build-test build-test: just -f {{justfile()}} build just -f {{justfile()}} test +preview-build-test: + just -f {{justfile()}} preview-build + just -f {{justfile()}} preview-test + # Run tests # just test-connect ubuntu2204 @@ -51,3 +95,22 @@ test-connect os="ubuntu2204": exit 1 fi bash -c "$build_cmd" + +# just test-connect ubuntu2204 +test-connect-preview os="ubuntu2204": + #!/bin/bash + set -x + just -f {{justfile()}} create-builder || true + build_cmd=$( \ + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + python3 {{justfile_directory()}}/tools/bake_test_command_extract.py connect-daily {{os}} --file "docker-bake.preview.hcl" \ + ) + echo $build_cmd + if [[ $? -ne 0 ]]; then + echo "Failed to extract build command" + exit 1 + fi + bash -c "$build_cmd" diff --git a/docker-bake.hcl b/docker-bake.hcl index 90e6aa70f..89b4fb5b1 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -354,7 +354,7 @@ target "package-manager" { target = "build" name = "package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" - tags = get_tags(builds.os, "package-manager", PACKAGE_MANAGER_VERSION) + tags = get_tags(builds.os, "rstudio-package-manager", PACKAGE_MANAGER_VERSION) dockerfile = "Dockerfile.${builds.os}" context = "package-manager" diff --git a/docker-bake.preview.hcl b/docker-bake.preview.hcl new file mode 100644 index 000000000..fac64c41e --- /dev/null +++ b/docker-bake.preview.hcl @@ -0,0 +1,617 @@ +### Variable definitions ### +variable BRANCH { + default = "dev" +} + +variable CONNECT_DAILY_VERSION { + default = null +} + +variable PACKAGE_MANAGER_DAILY_VERSION { + default = null +} + +variable WORKBENCH_DAILY_VERSION { + default = null +} + +variable WORKBENCH_PREVIEW_VERSION { + default = null +} + +variable DRIVERS_VERSION { + default = "2024.03.0" +} + +variable DEFAULT_QUARTO_VERSION { + default = "1.4.553" +} + +function tag_safe_version { + params = [version] + result = replace(version, "+", "-") +} + +function clean_version { + params = [version] + result = regex_replace(version, "[+|-].*", "") +} + +function get_tag_prefix { + params = [] + result = BRANCH != "main" ? "${BRANCH}-" : "" +} + +function get_drivers_version { + params = [os] + result = os == "centos7" ? "${DRIVERS_VERSION}-1" : DRIVERS_VERSION +} + +function get_os_alt_name { + params = [os] + result = os == "ubuntu2204" ? "jammy" : os +} + +function get_centos_tags { + params = [os, product, product_version, build_type] + result = [ + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + ] +} + +function get_ubuntu_tags { + params = [os, product, product_version, build_type] + result = [ + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${build_type}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${clean_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "ghcr.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${build_type}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${build_type}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${clean_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${clean_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${os}-${tag_safe_version(product_version)}", + "docker.io/rstudio/${product}:${get_tag_prefix()}${get_os_alt_name(os)}-${tag_safe_version(product_version)}", + ] +} + +# FIXME: There's an obnoxious amount of hardcoding in tag-related functions due to restrictions with what bake can actually interpret from HCL. +function get_tags { + params = [os, product, product_version, build_type] + result = os == "ubuntu2204" ? get_ubuntu_tags(os, product, product_version, build_type) : get_centos_tags(os, product, product_version, build_type) +} + +### Build matrices ### +variable BASE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable PRO_BUILD_MATRIX { + default = BASE_BUILD_MATRIX +} + +variable PACKAGE_MANAGER_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable CONNECT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable CONNECT_CONTENT_INIT_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204"}, + ] + } +} + +variable R_SESSION_COMPLETE_BUILD_MATRIX { + default = { + builds = [ + {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +variable WORKBENCH_BUILD_MATRIX { + default = { + builds = [ + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + ] + } +} + +### Group definitions ### +group "default" { + targets = [ + "product-base-dev", + "product-base-pro-dev", + "connect-daily", + "connect-content-init-daily", + "package-manager-daily", + "r-session-complete-preview", + "r-session-complete-daily", + "workbench-preview", + "workbench-daily", + ] +} + +group "build" { + targets = [ + "product-base-dev", + "product-base-pro-dev", + "connect-daily", + "connect-content-init-daily", + "package-manager-daily", + "r-session-complete-preview", + "r-session-complete-daily", + "workbench-preview", + "workbench-daily", + ] +} + +group "build-test" { + targets = [ + "product-base-dev", + "product-base-pro-dev", + "connect-daily", + "connect-content-init-daily", + "package-manager-daily", + "r-session-complete-preview", + "r-session-complete-daily", + "workbench-preview", + "workbench-daily", + "test-product-base-dev", + "test-product-base-pro-dev", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + "test-connect-content-init-daily", + "test-package-manager-daily", + "test-r-session-complete-daily", + "test-r-session-complete-preview", + "test-workbench-daily", + "test-workbench-preview", + ] +} + +group "test" { + targets = [ + "test-product-base-dev", + "test-product-base-pro-dev", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + "test-connect-content-init-daily", + "test-package-manager-daily", + "test-r-session-complete-daily", + "test-r-session-complete-preview", + "test-workbench-daily", + "test-workbench-preview", + ] +} + +group "base-dev-images" { + targets = [ + "product-base-dev", + "test-product-base-dev", + "product-base-pro-dev", + "test-product-base-pro-dev", + ] +} + +group "package-manager-daily-images" { + targets = [ + "package-manager-daily", + "test-package-manager-daily", + ] +} + +group "connect-daily-images" { + targets = [ + "connect-daily", + # "test-connect", # FIXME: This target requires a privileged environment which bake cannot provide + ] +} + +group "connect-content-init-daily-images" { + targets = [ + "connect-content-init-daily", + "test-connect-content-init-daily", + ] +} + +group "r-session-complete-images" { + targets = [ + "r-session-complete-daily", + "r-session-complete-preview", + "test-r-session-complete-daily", + "test-r-session-complete-preview", + ] +} + +group "workbench-images" { + targets = [ + "workbench-daily", + "workbench-preview", + "test-workbench-daily", + "test-workbench-preview", + ] +} + +### Dev Base Image targets ### +target "base" { + labels = { + "maintainer" = "Posit Docker " + } + output = ["type=image"] +} + +target "product-base-dev" { + inherits = ["base"] + target = "build" + + name = "product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/base" + + matrix = BASE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +target "test-product-base-dev" { + inherits = ["product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "test-product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] + + contexts = { + build = "target:product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = BASE_BUILD_MATRIX +} + +target "product-base-pro-dev" { + inherits = ["base"] + target = "build" + + name = "product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [ + "ghcr.io/rstudio/product-base-pro-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + "docker.io/rstudio/product-base-pro-dev:${builds.os}-r${builds.r_primary}_${builds.r_alternate}-py${builds.py_primary}_${builds.py_alternate}", + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "product/pro" + contexts = { + product-base = "target:product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + DRIVERS_VERSION = get_drivers_version(builds.os) + TINI_VERSION = "0.19.0" + QUARTO_VERSION = "1.3.340" + } +} + +target "test-product-base-pro-dev" { + inherits = ["product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"] + target = "test" + + name = "test-product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + tags = [] + + contexts = { + build = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PRO_BUILD_MATRIX +} + +### Package Manager targets ### +target "package-manager-daily" { + inherits = ["base"] + target = "build" + + name = "package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" + tags = get_tags(builds.os, "rstudio-package-manager-preview", PACKAGE_MANAGER_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "package-manager" + contexts = { + product-base = "target:product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = PACKAGE_MANAGER_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSPM_VERSION = PACKAGE_MANAGER_DAILY_VERSION + } +} + +target "test-package-manager-daily" { + inherits = ["package-manager-preview-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}"] + target = "test" + + name = "test-package-manager-preview-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" + tags = [] + + contexts = { + build = "target:package-manager-preview-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" + } + + matrix = PACKAGE_MANAGER_BUILD_MATRIX +} + +### Connect targets ### +target "connect-daily" { + inherits = ["base"] + target = "build" + + name = "connect-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect-preview", CONNECT_DAILY_VERSION, "daily") + + # We output Connect to OCI so it can be pulled in for testing later on. + output = [ + "type=image", + "type=oci,tar=false,dest=./.out/connect-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + ] + + dockerfile = "Dockerfile.${builds.os}" + context = "connect" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = CONNECT_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + RSC_VERSION = CONNECT_DAILY_VERSION + } +} + +# FIXME: This target requires a privileged environment which bake cannot provide +target "test-connect-daily" { + inherits = ["connect-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}"] + target = "test" + + name = "test-connect-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = [] + + contexts = { + build = "target:connect-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + } + + matrix = CONNECT_BUILD_MATRIX +} + +target "connect-content-init-daily" { + inherits = ["base"] + target = "build" + + name = "connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-connect-content-init-preview", CONNECT_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "connect-content-init" + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX + + args = { + RSC_VERSION = CONNECT_DAILY_VERSION + } +} + +target "test-connect-content-init" { + inherits = ["connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}"] + target = "test" + + name = "test-connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + tags = [] + + contexts = { + build = "target:connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" + } + + matrix = CONNECT_CONTENT_INIT_BUILD_MATRIX +} + +### Workbench targets ### +target "r-session-complete-daily" { + inherits = ["base"] + target = "build" + + name = "r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "r-session-complete-preview", WORKBENCH_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_DAILY_VERSION + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "test-r-session-complete-daily" { + inherits = ["r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}"] + target = "test" + + name = "test-r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + tags = [] + + contexts = { + build = "target:r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX +} + +target "r-session-complete-preview" { + inherits = ["base"] + target = "build" + + name = "r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = get_tags(builds.os, "r-session-complete-preview", WORKBENCH_PREVIEW_VERSION, "preview") + + dockerfile = "Dockerfile.${builds.os}" + context = "r-session-complete" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + JUPYTERLAB_VERSION = "3.6.5" + RSW_VERSION = WORKBENCH_PREVIEW_VERSION + RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" + RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "test-r-session-complete-preview" { + inherits = ["r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}"] + target = "test" + + name = "test-r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = [] + + contexts = { + build = "target:r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + } + + matrix = R_SESSION_COMPLETE_BUILD_MATRIX +} + +target "workbench-daily" { + inherits = ["base"] + + name = "workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench-preview", WORKBENCH_DAILY_VERSION, "daily") + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + RSW_VERSION = WORKBENCH_DAILY_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "test-workbench-daily" { + inherits = ["workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}"] + + name = "test-workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = [] + + contexts = { + build = "target:workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX +} + +target "workbench-preview" { + inherits = ["base"] + + name = "workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = get_tags(builds.os, "rstudio-workbench-preview", WORKBENCH_PREVIEW_VERSION, "preview") + + dockerfile = "Dockerfile.${builds.os}" + context = "workbench" + contexts = { + product-base-pro = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX + args = { + R_VERSION = builds.r_primary + R_VERSION_ALT = builds.r_alternate + PYTHON_VERSION = builds.py_primary + PYTHON_VERSION_ALT = builds.py_alternate + PYTHON_VERSION_JUPYTER = builds.py_alternate + RSW_VERSION = WORKBENCH_PREVIEW_VERSION + RSW_NAME = "rstudio-workbench" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + } +} + +target "test-workbench-preview" { + inherits = ["workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}"] + + name = "test-workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + tags = [] + + contexts = { + build = "target:workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + } + + matrix = WORKBENCH_BUILD_MATRIX +} diff --git a/tools/bake_test_command_extract.py b/tools/bake_test_command_extract.py index 733efd7a8..6bf67669b 100644 --- a/tools/bake_test_command_extract.py +++ b/tools/bake_test_command_extract.py @@ -12,10 +12,11 @@ parser.add_argument("image_name") parser.add_argument("operating_system") parser.add_argument("--builder", default="posit-builder") +parser.add_argument("--file", default="docker-bake.hcl") -def get_bake_plan(): - cmd = ["docker", "buildx", "bake", "-f", str(PROJECT_DIR / "docker-bake.hcl"), "--print"] +def get_bake_plan(bake_file="docker-bake.hcl"): + cmd = ["docker", "buildx", "bake", "-f", str(PROJECT_DIR / bake_file), "--print"] p = subprocess.run(cmd, capture_output=True) if p.returncode != 0: print(f"Failed to get bake plan: {p.stderr}") @@ -53,7 +54,7 @@ def build_test_command(target_name, target_spec, builder): def main(): args = parser.parse_args() - plan = get_bake_plan() + plan = get_bake_plan(args.file) targets = get_targets(plan, args.image_name, args.operating_system) for target_name, target_spec in targets.items(): cmd = build_test_command(target_name, target_spec, args.builder) From a3de06c9dc4c3ac3121e2a364dcd55dd5b528f5a Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 12:48:11 -0700 Subject: [PATCH 26/37] Update if clause for pushing preview images --- .github/workflows/build-bake-preview.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml index 61dd4c37c..6033f7e99 100644 --- a/.github/workflows/build-bake-preview.yaml +++ b/.github/workflows/build-bake-preview.yaml @@ -72,8 +72,14 @@ jobs: - name: Push id: push - if: github.ref == 'refs/heads/main' + if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} uses: docker/bake-action@v4 + env: + WORKBENCH_DAILY_VERSION: ${{ steps.get-version.outputs.WORKBENCH_DAILY_VERSION }} + WORKBENCH_PREVIEW_VERSION: ${{ steps.get-version.outputs.WORKBENCH_PREVIEW_VERSION }} + PACKAGE_MANAGER_DAILY_VERSION: ${{ steps.get-version.outputs.PACKAGE_MANAGER_DAILY_VERSION }} + CONNECT_DAILY_VERSION: ${{ steps.get-version.outputs.CONNECT_DAILY_VERSION }} + BRANCH: ${{ github.head_ref || github.ref_name }} with: set: | *.cache-from=type=gha From 59863b2534ad429506ef40f5ca5fd8e30372ebbf Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 12:59:09 -0700 Subject: [PATCH 27/37] Install requests package --- .github/workflows/build-bake-preview.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml index 6033f7e99..92d001282 100644 --- a/.github/workflows/build-bake-preview.yaml +++ b/.github/workflows/build-bake-preview.yaml @@ -38,6 +38,10 @@ jobs: with: python-version: '3.12' + - name: Install Python dependencies + run: | + pip install requests + - name: Get Version id: get-version run: | From 09dac6f85b66fa7ef8320d43e62c8e2a77e8baec Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 13:00:32 -0700 Subject: [PATCH 28/37] Provision buildx in preview --- .github/workflows/build-bake-preview.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml index 92d001282..31039d74a 100644 --- a/.github/workflows/build-bake-preview.yaml +++ b/.github/workflows/build-bake-preview.yaml @@ -34,6 +34,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - uses: actions/setup-python@v5 with: python-version: '3.12' From dccb4bb09699cb517ff152fcfa22c3947c4f77bf Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 14:24:34 -0700 Subject: [PATCH 29/37] Set file on preview builds --- .github/workflows/build-bake-preview.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml index 31039d74a..0cddb970f 100644 --- a/.github/workflows/build-bake-preview.yaml +++ b/.github/workflows/build-bake-preview.yaml @@ -72,6 +72,7 @@ jobs: *.cache-to=type=gha,compression=zstd targets: "build-test" push: false + files: docker-bake.preview.hcl - name: Test Connect (privileged container) run: | @@ -93,4 +94,4 @@ jobs: *.cache-to=type=gha,compression=zstd targets: "build" push: true - + files: docker-bake.preview.hcl From 8ce3a4776407a2b73dbdb08f39e41bff1658da7f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 14:29:06 -0700 Subject: [PATCH 30/37] Fix target names --- docker-bake.preview.hcl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-bake.preview.hcl b/docker-bake.preview.hcl index fac64c41e..cc3bc0522 100644 --- a/docker-bake.preview.hcl +++ b/docker-bake.preview.hcl @@ -370,14 +370,14 @@ target "package-manager-daily" { } target "test-package-manager-daily" { - inherits = ["package-manager-preview-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}"] + inherits = ["package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}"] target = "test" - name = "test-package-manager-preview-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" + name = "test-package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" tags = [] contexts = { - build = "target:package-manager-preview-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" + build = "target:package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" } matrix = PACKAGE_MANAGER_BUILD_MATRIX @@ -445,7 +445,7 @@ target "connect-content-init-daily" { } } -target "test-connect-content-init" { +target "test-connect-content-init-daily" { inherits = ["connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}"] target = "test" @@ -566,13 +566,13 @@ target "workbench-daily" { } target "test-workbench-daily" { - inherits = ["workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}"] + inherits = ["workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}"] - name = "test-workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + name = "test-workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" tags = [] contexts = { - build = "target:workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" + build = "target:workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" } matrix = WORKBENCH_BUILD_MATRIX From 8026d4e3c5228d1b9c7fe976148eafea9071e67f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 14:33:17 -0700 Subject: [PATCH 31/37] Fix CentOS 7 image bad ref --- r-session-complete/Dockerfile.centos7 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r-session-complete/Dockerfile.centos7 b/r-session-complete/Dockerfile.centos7 index 2165e0d58..4d5c08bf3 100644 --- a/r-session-complete/Dockerfile.centos7 +++ b/r-session-complete/Dockerfile.centos7 @@ -4,7 +4,7 @@ ARG PYTHON_VERSION=3.9.14 ARG PYTHON_VERSION_ALT=3.8.15 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io -FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} as build +FROM product-base-pro as build LABEL maintainer="RStudio Docker "` ARG R_VERSION=4.2.3 From 8ccd22678be4cc8ad7b4d04acf8571a0f403b9f3 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 Apr 2024 14:51:58 -0700 Subject: [PATCH 32/37] Calculate RSW download URL in bake --- docker-bake.preview.hcl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docker-bake.preview.hcl b/docker-bake.preview.hcl index cc3bc0522..79dcccbcc 100644 --- a/docker-bake.preview.hcl +++ b/docker-bake.preview.hcl @@ -27,6 +27,15 @@ variable DEFAULT_QUARTO_VERSION { default = "1.4.553" } +variable RSW_PREVIEW_URL_BASE { + default = "https://s3.amazonaws.com/rstudio-ide-build/server/" +} + +function get_rsw_download_url { + params = [os] + result = os == "centos7" ? "${RSW_PREVIEW_URL_BASE}centos7/x86_64" : "${RSW_PREVIEW_URL_BASE}jammy/amd64" +} + function tag_safe_version { params = [version] result = replace(version, "+", "-") @@ -482,7 +491,7 @@ target "r-session-complete-daily" { JUPYTERLAB_VERSION = "3.6.5" RSW_VERSION = WORKBENCH_DAILY_VERSION RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" - RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) } } @@ -522,7 +531,7 @@ target "r-session-complete-preview" { JUPYTERLAB_VERSION = "3.6.5" RSW_VERSION = WORKBENCH_PREVIEW_VERSION RSW_NAME = builds.os == "centos7" ? "rstudio-workbench-rhel" : "rstudio-workbench" - RSW_DOWNLOAD_URL = builds.os == "centos7" ? "https://s3.amazonaws.com/rstudio-ide-build/server/centos7/x86_64" : "https://download2.rstudio.org/server/jammy/amd64" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) } } @@ -561,7 +570,7 @@ target "workbench-daily" { PYTHON_VERSION_JUPYTER = builds.py_alternate RSW_VERSION = WORKBENCH_DAILY_VERSION RSW_NAME = "rstudio-workbench" - RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) } } @@ -599,7 +608,7 @@ target "workbench-preview" { PYTHON_VERSION_JUPYTER = builds.py_alternate RSW_VERSION = WORKBENCH_PREVIEW_VERSION RSW_NAME = "rstudio-workbench" - RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" + RSW_DOWNLOAD_URL = get_rsw_download_url(builds.os) } } From 004006cc7f5c205a8cd67fa13887bd0f47e59c6f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 9 Apr 2024 08:16:48 -0700 Subject: [PATCH 33/37] Wait for launcher startup in Workbench tests --- workbench/test/run_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/workbench/test/run_tests.sh b/workbench/test/run_tests.sh index 76433623f..2110c47d2 100755 --- a/workbench/test/run_tests.sh +++ b/workbench/test/run_tests.sh @@ -12,6 +12,7 @@ echo "--> Starting RStudio Workbench" echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT +wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} From 74e3f54acb3136705eb3408a1f13f69db9465f30 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 9 Apr 2024 09:59:55 -0700 Subject: [PATCH 34/37] Add wait time in Workbench test runner to ensure services are up --- workbench/test/run_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workbench/test/run_tests.sh b/workbench/test/run_tests.sh index 2110c47d2..db5b4c3d7 100755 --- a/workbench/test/run_tests.sh +++ b/workbench/test/run_tests.sh @@ -9,15 +9,17 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & +sleep 5 echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" +sleep 5 GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} GOSS_VERSION=${GOSS_VERSION:-0.4.6} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-10} # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ From 427b8958c3aa8003d4b6e07bcc824a413121bdbe Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 9 Apr 2024 11:06:40 -0700 Subject: [PATCH 35/37] Do not cache or output tests Use a standard runner for content images --- .github/workflows/build-bake.yaml | 2 +- docker-bake.hcl | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 5334f0834..6b59b5fc6 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -57,7 +57,7 @@ jobs: push: true build-content: - runs-on: ubuntu-latest-4x + runs-on: ubuntu-latest concurrency: group: bake-content-${{ github.ref }} diff --git a/docker-bake.hcl b/docker-bake.hcl index 89b4fb5b1..c2ce91ccc 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -298,6 +298,8 @@ target "test-product-base" { name = "test-product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:product-base-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" @@ -340,6 +342,8 @@ target "test-product-base-pro" { name = "test-product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" @@ -378,6 +382,8 @@ target "test-package-manager" { name = "test-package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:package-manager-${builds.os}-${replace(PACKAGE_MANAGER_VERSION, ".", "-")}" @@ -423,6 +429,8 @@ target "test-connect" { name = "test-connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:connect-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" @@ -454,6 +462,8 @@ target "test-connect-content-init" { name = "test-connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:connect-content-init-${builds.os}-${replace(CONNECT_VERSION, ".", "-")}" @@ -495,6 +505,8 @@ target "test-r-session-complete" { name = "test-r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:r-session-complete-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" @@ -533,6 +545,8 @@ target "test-workbench" { name = "test-workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:workbench-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" @@ -581,6 +595,8 @@ target "test-workbench-for-google-cloud-workstations" { name = "test-workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:workbench-for-google-cloud-workstation-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" @@ -635,6 +651,8 @@ target "test-workbench-for-microsoft-azure-ml" { target = "test" name = "test-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" + output = [] + no-cache = true contexts = { build = "target:build-workbench-for-microsoft-azure-ml-${builds.os}-${replace(workbench_version_clean(), ".", "-")}" From 75534503fd783dcdb127f062318311e7fb8d4bc1 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 9 Apr 2024 11:56:16 -0700 Subject: [PATCH 36/37] Do not cache or output tests for preview Provide a buildx config for GHA Unify test sleep behavior of Workbench images --- .github/workflows/build-bake-preview.yaml | 2 ++ .github/workflows/build-bake.yaml | 6 +++++- docker-bake.preview.hcl | 18 ++++++++++++++++++ share/buildkitd.toml | 1 + share/local_buildkitd.toml | 2 ++ .../test/run_tests.sh | 2 +- .../test/run_tests.sh | 2 +- workbench/test/run_tests.sh | 5 ++--- 8 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 share/buildkitd.toml diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml index 0cddb970f..85a0b5116 100644 --- a/.github/workflows/build-bake-preview.yaml +++ b/.github/workflows/build-bake-preview.yaml @@ -36,6 +36,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + buildkitd-config: ./share/buildkitd.toml - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 6b59b5fc6..c78a7d030 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -30,6 +30,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + buildkitd-config: ./share/buildkitd.toml - name: Build and test id: build @@ -57,7 +59,7 @@ jobs: push: true build-content: - runs-on: ubuntu-latest + runs-on: ubuntu-latest-4x concurrency: group: bake-content-${{ github.ref }} @@ -69,6 +71,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + buildkitd-config: ./share/buildkitd.toml - name: Build id: build diff --git a/docker-bake.preview.hcl b/docker-bake.preview.hcl index 79dcccbcc..be8440e97 100644 --- a/docker-bake.preview.hcl +++ b/docker-bake.preview.hcl @@ -304,6 +304,8 @@ target "test-product-base-dev" { name = "test-product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:product-base-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" @@ -346,6 +348,8 @@ target "test-product-base-pro-dev" { name = "test-product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:product-base-pro-dev-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" @@ -384,6 +388,8 @@ target "test-package-manager-daily" { name = "test-package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:package-manager-daily-${builds.os}-${replace(PACKAGE_MANAGER_DAILY_VERSION, ".", "-")}" @@ -429,6 +435,8 @@ target "test-connect-daily" { name = "test-connect-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:connect-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" @@ -460,6 +468,8 @@ target "test-connect-content-init-daily" { name = "test-connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:connect-content-init-daily-${builds.os}-${replace(tag_safe_version(CONNECT_DAILY_VERSION), ".", "-")}" @@ -501,6 +511,8 @@ target "test-r-session-complete-daily" { name = "test-r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:r-session-complete-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" @@ -541,6 +553,8 @@ target "test-r-session-complete-preview" { name = "test-r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:r-session-complete-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" @@ -579,6 +593,8 @@ target "test-workbench-daily" { name = "test-workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:workbench-daily-${builds.os}-${replace(tag_safe_version(WORKBENCH_DAILY_VERSION), ".", "-")}" @@ -617,6 +633,8 @@ target "test-workbench-preview" { name = "test-workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" tags = [] + output = [] + no-cache = true contexts = { build = "target:workbench-preview-${builds.os}-${replace(tag_safe_version(WORKBENCH_PREVIEW_VERSION), ".", "-")}" diff --git a/share/buildkitd.toml b/share/buildkitd.toml new file mode 100644 index 000000000..ac746f6f2 --- /dev/null +++ b/share/buildkitd.toml @@ -0,0 +1 @@ +insecure-entitlements = ["security.insecure"] \ No newline at end of file diff --git a/share/local_buildkitd.toml b/share/local_buildkitd.toml index 531aab30f..a775e7ff7 100644 --- a/share/local_buildkitd.toml +++ b/share/local_buildkitd.toml @@ -1,2 +1,4 @@ +insecure-entitlements = ["security.insecure"] + [worker.oci] max-parallelism = 4 \ No newline at end of file diff --git a/workbench-for-google-cloud-workstations/test/run_tests.sh b/workbench-for-google-cloud-workstations/test/run_tests.sh index 10d390043..72f4c1c6e 100755 --- a/workbench-for-google-cloud-workstations/test/run_tests.sh +++ b/workbench-for-google-cloud-workstations/test/run_tests.sh @@ -9,7 +9,7 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & -sleep 10 +sleep 15 netstat -tulpn diff --git a/workbench-for-microsoft-azure-ml/test/run_tests.sh b/workbench-for-microsoft-azure-ml/test/run_tests.sh index daa29c97f..894196850 100755 --- a/workbench-for-microsoft-azure-ml/test/run_tests.sh +++ b/workbench-for-microsoft-azure-ml/test/run_tests.sh @@ -19,7 +19,7 @@ echo "--> Startup complete" GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} GOSS_VARS_FILE=${GOSS_VARS_FILE:-/tmp/test/goss_vars.yaml} GOSS_VERSION=${GOSS_VERSION:-0.4.6} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5} # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ diff --git a/workbench/test/run_tests.sh b/workbench/test/run_tests.sh index db5b4c3d7..8304fec0f 100755 --- a/workbench/test/run_tests.sh +++ b/workbench/test/run_tests.sh @@ -9,17 +9,16 @@ trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/start # start rstudio-server echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & -sleep 5 +sleep 15 echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" -sleep 5 GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} GOSS_VERSION=${GOSS_VERSION:-0.4.6} -GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-10} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5} # install goss to tmp location and make executable curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ From c9e871978debc8e2679f40573f252bd9895fae16 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 9 Apr 2024 12:10:20 -0700 Subject: [PATCH 37/37] Show running processes (debugging) --- workbench/test/run_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workbench/test/run_tests.sh b/workbench/test/run_tests.sh index 8304fec0f..00bba3847 100755 --- a/workbench/test/run_tests.sh +++ b/workbench/test/run_tests.sh @@ -16,6 +16,8 @@ wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" +ps aux | grep rs + GOSS_FILE=${GOSS_FILE:-/tmp/test/goss.yaml} GOSS_VERSION=${GOSS_VERSION:-0.4.6} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-5}