Extract common operations to reusable scripts in base images (#716)

* Fix/exclude goss tests for buildx bake * Implement separate test for Connect * Implement bake preview builds * Provision buildx in preview * Do not cache or output tests for preview Provide a buildx config for GHA Unify test sleep behavior of Workbench images * Remove test targets * Remove test layers from Dockerfiles * Change test running to be orchestrated by a Python script * Update preview builds for new test script * Create common use scripts in base image for Ubuntu 22.04 images * Create common use scripts in base image for CentOS 7 images * Move deps to subdirectory Fix hadolint issues * Ignore hadolint issue * Update permissions hex for tests * Remove build.justfile * Remove unnecessary buildx setup from build-bake-preview.yaml * Move libarchive-dev to ubuntu2204_packages.txt
rstudio · Apr 18, 2024 · 0d64e87 · 0d64e87
1 parent 19c6dd3
commit 0d64e87
Show file tree

Hide file tree

Showing 19 changed files with 1,203 additions and 160 deletions.
diff --git a/product/base/Dockerfile.centos7 b/product/base/Dockerfile.centos7
@@ -8,30 +8,17 @@ ARG PYTHON_VERSION_ALT=3.8.15
 ARG TINI_VERSION=0.19.0
 ARG QUARTO_VERSION=1.3.340
 
+ARG SCRIPTS_DIR=/opt/positscripts
+
+COPY scripts/rhel/* ${SCRIPTS_DIR}/
+
 ### Update/upgrade system packages ###
-RUN yum upgrade -y -q \
-    && yum install -y -q \
-      epel-release \
-    && yum install -y -q \
-      bzip2 \
-      git \
-      gpg \
-      gpg-agent \
-      libcurl-devel \
-      libuser-devel \
-      libxml2-devel \
-      openssl-devel \
-      openssh-clients \
-      pandoc \
-      perl-Digest-MD5 \
-      postgresql-libs \
-      rrdtool \
-      sudo \
-      unixODBC \
-      unixODBC-devel \
-      wget \
-      which \
-    && yum clean all
+COPY deps/centos7_packages.txt /tmp/packages.txt
+# hadolint ignore=SC2046
+RUN ${SCRIPTS_DIR}/yum.sh --update upgrade \
+    && ${SCRIPTS_DIR}/yum.sh install epel-release \
+    && ${SCRIPTS_DIR}/yum.sh install $(cat /tmp/packages.txt) \
+    && ${SCRIPTS_DIR}/yum.sh --clean
 
 ### Install tini ###
 ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64 /tini
@@ -50,45 +37,29 @@ RUN curl -sL "https://yihui.org/tinytex/install-bin-unix.sh" | sh \
     && /opt/TinyTeX/bin/*/tlmgr path add
 
 ### Install Quarto ###
-RUN curl -o quarto.tar.gz -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz \
-    && mkdir -p /opt/quarto/${QUARTO_VERSION} \
-    && tar -zxvf quarto.tar.gz -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 \
-    && rm -f quarto.tar.gz \
+RUN QUARTO_VERSION=${QUARTO_VERSION} ${SCRIPTS_DIR}/install_quarto.sh \
     && ln -s /opt/quarto/${QUARTO_VERSION}/bin/quarto /usr/local/bin/quarto
 
 ### Install R versions ###
-RUN curl -O https://cdn.rstudio.com/r/centos-7/pkgs/R-${R_VERSION}-1-1.x86_64.rpm \
-    && curl -O https://cdn.rstudio.com/r/centos-7/pkgs/R-${R_VERSION_ALT}-1-1.x86_64.rpm \
-    && yum install -y R-${R_VERSION}-1-1.x86_64.rpm \
-    && yum install -y R-${R_VERSION_ALT}-1-1.x86_64.rpm \
-    && yum clean all \
-    && rm -rf R-${R_VERSION}-1-1.x86_64.rpm \
-    && rm -rf R-${R_VERSION_ALT}-1-1.x86_64.rpm \
+RUN R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh \
+    && R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh \
     && ln -s /opt/R/${R_VERSION} /opt/R/default \
     && ln -s /opt/R/default/bin/R /usr/local/bin/R \
     && ln -s /opt/R/default/bin/Rscript /usr/local/bin/Rscript
 
 ### Install Python versions ###
-RUN curl -O https://cdn.rstudio.com/python/centos-7/pkgs/python-${PYTHON_VERSION}-1-1.x86_64.rpm \
-    && curl -O https://cdn.rstudio.com/python/centos-7/pkgs/python-${PYTHON_VERSION_ALT}-1-1.x86_64.rpm \
-    && yum install -y python-${PYTHON_VERSION}-1-1.x86_64.rpm \
-    && yum install -y python-${PYTHON_VERSION_ALT}-1-1.x86_64.rpm \
-    && yum clean all \
-    && rm -rf python-${PYTHON_VERSION}-1-1.x86_64.rpm \
-    && rm -rf python-${PYTHON_VERSION_ALT}-1-1.x86_64.rpm \
-    && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \
-    && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \
-    && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \
-    && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \
-    && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default
+COPY deps/requirements.txt /tmp/requirements.txt
+RUN PYTHON_VERSION=${PYTHON_VERSION} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \
+    && PYTHON_VERSION=${PYTHON_VERSION_ALT} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \
+    && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default \
+    && rm -f /tmp/requirements.txt
 
 ### Locale configuration ###
 RUN localedef -i en_US -f UTF-8 en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
-
 LABEL posit.r.version="${R_VERSION}" \
       posit.r.version_alt="${R_VERSION_ALT}" \
       posit.python.version="${PYTHON_VERSION}" \

diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204
@@ -9,74 +9,18 @@ ARG PYTHON_VERSION_ALT=3.8.17
 ARG TINI_VERSION=0.19.0
 ARG QUARTO_VERSION=1.3.340
 
+ARG SCRIPTS_DIR=/opt/positscripts
+
+COPY scripts/ubuntu/* ${SCRIPTS_DIR}/
+
 ### Update/upgrade system packages ###
-RUN apt-get update --fix-missing  \
-    && apt-get upgrade -yq \
-    && apt-get install -yq --no-install-recommends \
-        apt-transport-https \
-        ca-certificates \
-        cmake \
-        cracklib-runtime \
-        curl \
-        default-jdk \
-        dirmngr \
-        dpkg-sig \
-        g++ \
-        gcc \
-        gdal-bin \
-        gfortran \
-        git \
-        gpg \
-        gpg-agent \
-        gsfonts \
-        imagemagick \
-        libarchive-dev \
-        libcairo2-dev \
-        libcurl4-openssl-dev \
-        libev-dev \
-        libfontconfig1-dev \
-        libfreetype6-dev \
-        libfribidi-dev \
-        libgdal-dev \
-        libgeos-dev \
-        libgl1-mesa-dev \
-        libglpk-dev \
-        libglu1-mesa-dev \
-        libgmp3-dev \
-        libharfbuzz-dev \
-        libicu-dev \
-        libjpeg-dev \
-        libmagick++-dev \
-        libmysqlclient-dev \
-        libopenblas-dev \
-        libpaper-utils \
-        libpcre2-dev \
-        libpng-dev \
-        libproj-dev \
-        libsodium-dev \
-        libssh2-1-dev \
-        libssl-dev \
-        libtiff-dev \
-        libudunits2-dev \
-        libv8-dev \
-        libxml2-dev \
-        locales \
-        make \
-        openssh-client \
-        pandoc \
-        perl \
-        sudo \
-        tcl \
-        tk \
-        tk-dev \
-        tk-table \
-        tzdata \
-        unixodbc-dev \
-        unzip \
-        wget \
-        zip \
-        zlib1g-dev \
-    && rm -rf /var/lib/apt/lists/*
+COPY deps/ubuntu2204_packages.txt /tmp/apt_packages.txt
+# hadolint ignore=SC2046
+RUN ${SCRIPTS_DIR}/apt.sh --update install lsof \
+    && ${SCRIPTS_DIR}/apt.sh --update upgrade \
+    && ${SCRIPTS_DIR}/apt.sh install $(cat /tmp/apt_packages.txt) \
+    && ${SCRIPTS_DIR}/apt.sh --clean \
+    && rm -f /tmp/apt_packages.txt
 
 ### Install tini ###
 ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64 /tini
@@ -95,35 +39,22 @@ RUN curl -sL "https://yihui.org/tinytex/install-bin-unix.sh" | sh \
     && /opt/TinyTeX/bin/*/tlmgr path add
 
 ### Install Quarto ###
-RUN curl -o quarto.tar.gz -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz \
-    && mkdir -p /opt/quarto/${QUARTO_VERSION} \
-    && tar -zxvf quarto.tar.gz -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 \
-    && rm -f quarto.tar.gz \
+RUN QUARTO_VERSION=${QUARTO_VERSION} ${SCRIPTS_DIR}/install_quarto.sh \
     && ln -s /opt/quarto/${QUARTO_VERSION}/bin/quarto /usr/local/bin/quarto
 
 ### Install R versions ###
-RUN curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION}_1_amd64.deb \
-    && curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \
-    && apt-get install -yq --no-install-recommends ./r-${R_VERSION}_1_amd64.deb \
-    && apt-get install -yq --no-install-recommends ./r-${R_VERSION_ALT}_1_amd64.deb \
-    && rm -f ./r-${R_VERSION}_1_amd64.deb \
-    && rm -f ./r-${R_VERSION_ALT}_1_amd64.deb \
+RUN R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh \
+    && R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh \
     && ln -s /opt/R/${R_VERSION} /opt/R/default \
     && ln -s /opt/R/default/bin/R /usr/local/bin/R \
     && ln -s /opt/R/default/bin/Rscript /usr/local/bin/Rscript
 
 ### Install Python versions ###
-RUN curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \
-    && curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \
-    && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION}_1_amd64.deb \
-    && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION_ALT}_1_amd64.deb \
-    && rm -rf python-${PYTHON_VERSION}_1_amd64.deb \
-    && rm -rf python-${PYTHON_VERSION_ALT}_1_amd64.deb \
-    && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \
-    && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \
-    && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \
-    && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \
-    && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default
+COPY deps/requirements.txt /tmp/requirements.txt
+RUN PYTHON_VERSION=${PYTHON_VERSION} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \
+    && PYTHON_VERSION=${PYTHON_VERSION_ALT} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \
+    && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default \
+    && rm -f /tmp/requirements.txt
 
 ### Locale configuration ###
 RUN localedef -i en_US -f UTF-8 en_US.UTF-8
@@ -132,13 +63,6 @@ ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 ENV TZ=UTC
 
-### Clean up ###
-RUN apt-get install -yqf --no-install-recommends \
-    && apt-get autoremove \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-
 LABEL posit.r.version="${R_VERSION}" \
       posit.r.version_alt="${R_VERSION_ALT}" \
       posit.python.version="${PYTHON_VERSION}" \

diff --git a/product/base/deps/centos7_packages.txt b/product/base/deps/centos7_packages.txt
@@ -0,0 +1,20 @@
+bzip2
+curl
+git
+gpg
+gpg-agent
+libcurl-devel
+libuser-devel
+libxml2-devel
+openssl-devel
+openssh-clients
+pandoc
+perl-Digest-MD5
+postgresql-libs
+redhat-lsb-core
+rrdtool
+sudo
+unixODBC
+unixODBC-devel
+wget
+which
diff --git a/product/base/deps/requirements.txt b/product/base/deps/requirements.txt
@@ -0,0 +1,2 @@
+virtualenv<20
+setuptools
diff --git a/product/base/deps/ubuntu2204_packages.txt b/product/base/deps/ubuntu2204_packages.txt
@@ -0,0 +1,66 @@
+apt-transport-https
+build-essential
+ca-certificates
+cmake
+cracklib-runtime
+curl
+default-jdk
+dirmngr
+dpkg-sig
+g++
+gcc
+gdal-bin
+gfortran
+git
+gnupg2
+gpg
+gpg-agent
+gsfonts
+imagemagick
+libarchive-dev
+libcairo2-dev
+libcurl4-openssl-dev
+libev-dev
+libfontconfig1-dev
+libfreetype6-dev
+libfribidi-dev
+libgdal-dev
+libgeos-dev
+libgl1-mesa-dev
+libglpk-dev
+libglu1-mesa-dev
+libgmp3-dev
+libharfbuzz-dev
+libicu-dev
+libjpeg-dev
+libmagick++-dev
+libmysqlclient-dev
+libopenblas-dev
+libpaper-utils
+libpcre2-dev
+libpng-dev
+libproj-dev
+libsodium-dev
+libssh2-1-dev
+libssl-dev
+libtiff-dev
+libudunits2-dev
+libv8-dev
+libxml2-dev
+locales
+lsb-release
+make
+openssh-client
+pandoc
+perl
+sudo
+tcl
+tk
+tk-dev
+tk-table
+tzdata
+unixodbc-dev
+unzip
+wget
+zip
+zlib1g-dev
diff --git a/product/base/scripts/rhel/install_drivers.sh b/product/base/scripts/rhel/install_drivers.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -exo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+# Output delimiter
+d="===="
+
+if [ -z "$DRIVERS_VERSION" ]; then
+    echo "$d No DRIVERS_VERSION specified $d"
+    exit 1
+fi
+
+echo "$d$d Installing Professional Drivers ${DRIVERS_VERSION} $d$d"
+
+drivers_url="https://cdn.rstudio.com/drivers/7C152C12/installer/rstudio-drivers-${DRIVERS_VERSION}.el.x86_64.rpm"
+curl -sL "$drivers_url" -o "/tmp/rstudio-drivers_${DRIVERS_VERSION}.el.x86_64.rpm"
+
+yum install -y -q "/tmp/rstudio-drivers_${DRIVERS_VERSION}.el.x86_64.rpm"
+cat /opt/rstudio-drivers/odbcinst.ini.sample > /etc/odbcinst.ini
+
+rm /tmp/rstudio-drivers_${DRIVERS_VERSION}.el.x86_64.rpm