From f2db0fa5081559fc8c1aee6c38f9079db293f840 Mon Sep 17 00:00:00 2001 From: Skye Turriff Date: Fri, 15 Nov 2024 14:25:56 -0500 Subject: [PATCH 1/4] Update build-manual.yaml Adds the WORKBENCH_SESSION_INIT variable to the manual build flow --- .github/workflows/build-manual.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index 5900804b..e5980f89 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -141,6 +141,8 @@ jobs: product="CONNECT" elif [[ "$product" == "package-manager" ]]; then product="PACKAGE_MANAGER" + elif [[ "$product" == "workbench-session-init" ]]; then + product="WORKBENCH_SESSION_INIT" else product="WORKBENCH" fi From 644d2d4067b4fa72261e436bec131f382d40454e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 20 Nov 2024 10:47:48 -0700 Subject: [PATCH 2/4] Merge versions job into workbench-session-init job --- .github/workflows/build-bake.yaml | 55 ++++++++++++------------------- Justfile | 3 +- 2 files changed, 22 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index e2d9f539..2fc69290 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -37,39 +37,6 @@ jobs: GIT_SHA=$(git rev-parse --short HEAD) echo "GIT_SHA=$GIT_SHA" >> $GITHUB_OUTPUT echo "$GIT_SHA" - versions: - name: Fetch Workbench session init container version - runs-on: ubuntu-latest - - concurrency: - group: fetch-versions-${{ github.ref }} - cancel-in-progress: true - - outputs: - WORKBENCH_SESSION_INIT_VERSION: ${{ steps.get-version.outputs.WORKBENCH_SESSION_INIT_VERSION }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Just - uses: extractions/setup-just@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install Python dependencies - run: | - pip install requests - - - name: Get Version - id: get-version - run: | - WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) - echo "WORKBENCH_SESSION_INIT_VERSION=$WORKBENCH_SESSION_INIT_VERSION" >> $GITHUB_OUTPUT base: needs: [setup] @@ -335,7 +302,6 @@ jobs: env: target: workbench-session-init GIT_SHA: ${{ needs.setup.outputs.GIT_SHA }} - WORKBENCH_SESSION_INIT_VERSION: ${{ needs.versions.outputs.WORKBENCH_SESSION_INIT_VERSION }} steps: - name: Checkout @@ -354,7 +320,28 @@ jobs: with: buildkitd-config: ./share/buildkitd.toml + - name: Set up Just + uses: extractions/setup-just@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Python dependencies + run: | + pip install requests + + - name: Get Version + id: get-version + run: | + WORKBENCH_SESSION_INIT_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) + echo "WORKBENCH_SESSION_INIT_VERSION=$WORKBENCH_SESSION_INIT_VERSION" >> $GITHUB_OUTPUT + - name: Build, Test, and Push + env: + WORKBENCH_SESSION_INIT_VERSION: ${{ steps.get-version.outputs.WORKBENCH_SESSION_INIT_VERSION }} uses: ./.github/actions/bake-test-push with: target: ${{ env.target }} diff --git a/Justfile b/Justfile index a742d7e5..56ed3e3a 100644 --- a/Justfile +++ b/Justfile @@ -59,8 +59,7 @@ bake target="default": docker buildx bake --builder=posit-builder -f docker-bake.hcl {{target}} # just preview-bake workbench-images dev -preview-build: - just preview-bake "default" +alias preview-build := preview-bake preview-bake target branch="$(git branch --show-current)": just -f {{justfile()}} create-builder || true WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ From b04cd5d2d3feb9a203d7c62bb357907aaa816e9f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 20 Nov 2024 11:10:05 -0700 Subject: [PATCH 3/4] Remove versions as dependency job --- .github/workflows/build-bake.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 2fc69290..a2c11415 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -291,7 +291,7 @@ jobs: snyk-token: '${{ secrets.SNYK_TOKEN }}' workbench-session-init: - needs: [setup, versions] + needs: [setup] name: Workbench Session Init runs-on: ubuntu-latest-8x From 6363aefadec67cf8ee9db1119d7a6452b109026b Mon Sep 17 00:00:00 2001 From: Skye Turriff Date: Wed, 20 Nov 2024 15:12:19 -0500 Subject: [PATCH 4/4] Add Go-based entrypoint for Workbench session init (#871) * add go program * add multi-stage build dockerfile * add positron * fix path * update err msgs and reuse file info * merge comments * enhance file copying with otiai10/copy package and update Dockerfile * remove uneeded components * add shared_run to common deps * fix component filename * update Dockerfile to use ubuntu:22.04 base image, as the scratch image doesn't work nicely with GOSS * remove test var --------- Co-authored-by: Ian Pittwood --- workbench-session-init/Dockerfile.ubuntu2204 | 39 +++- workbench-session-init/entrypoint/go.mod | 9 + workbench-session-init/entrypoint/go.sum | 6 + workbench-session-init/entrypoint/main.go | 170 ++++++++++++++++++ .../entrypoint/main_test.go | 149 +++++++++++++++ workbench-session-init/run.sh | 17 -- workbench-session-init/test/goss.yaml | 2 +- 7 files changed, 368 insertions(+), 24 deletions(-) create mode 100644 workbench-session-init/entrypoint/go.mod create mode 100644 workbench-session-init/entrypoint/go.sum create mode 100644 workbench-session-init/entrypoint/main.go create mode 100644 workbench-session-init/entrypoint/main_test.go delete mode 100644 workbench-session-init/run.sh diff --git a/workbench-session-init/Dockerfile.ubuntu2204 b/workbench-session-init/Dockerfile.ubuntu2204 index 2d0bbc44..7b8ab4f7 100644 --- a/workbench-session-init/Dockerfile.ubuntu2204 +++ b/workbench-session-init/Dockerfile.ubuntu2204 @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 AS build +FROM ubuntu:22.04 AS builder # Install required tools: # - ca-certificates installs necessary certificates to use cURL with HTTPS websites @@ -9,17 +9,44 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* ARG RSW_VERSION=2024.09.1+394.pro7 +ARG GO_VERSION=1.22.2 +# Download the RStudio Workbench session components and install Go SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN mkdir -p /pwb-staging && \ RSW_VERSION_URL=$(echo -n "${RSW_VERSION}" | sed 's/+/-/g') && \ - echo "Downloading https://s3.amazonaws.com/rstudio-ide-build/session/multi/x86_64/rsp-session-multi-linux-${RSW_VERSION_URL}-x86_64.tar.gz" && \ curl -fsSL -o /pwb-staging/rsp-session-multi-linux.tar.gz "https://s3.amazonaws.com/rstudio-ide-build/session/multi/x86_64/rsp-session-multi-linux-${RSW_VERSION_URL}-x86_64.tar.gz" && \ mkdir -p /opt/session-components && \ - tar -C /opt/session-components -xf /pwb-staging/rsp-session-multi-linux.tar.gz && \ - chmod -R 755 /opt/session-components && \ + tar -C /opt/session-components -xpf /pwb-staging/rsp-session-multi-linux.tar.gz && \ + chmod 755 /opt/session-components && \ + curl -fsSL -o /pwb-staging/go.tar.gz "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" && \ + tar -C /usr/local -xf /pwb-staging/go.tar.gz && \ rm -rf /pwb-staging -COPY --chmod=755 run.sh /usr/local/bin/run.sh +# Add Go binary to PATH +ENV PATH="/usr/local/go/bin:$PATH" + +# Set the Go workspace +WORKDIR /workspace + +# Copy the Go source code and download dependencies +COPY entrypoint/go.mod entrypoint/go.sum ./ +RUN go mod download + +# Copy the Go source code and build the binary +COPY entrypoint/main.go . +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags '-s -w' -o entrypoint main.go + +# Create the final image +FROM ubuntu:22.04 AS build + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* + +# Copy the compiled Go binary and session components from the builder stage +COPY --from=builder --chmod=755 /workspace/entrypoint /usr/local/bin/entrypoint +COPY --from=builder --chmod=755 /opt/session-components /opt/session-components -ENTRYPOINT ["/usr/local/bin/run.sh"] +ENTRYPOINT ["/usr/local/bin/entrypoint"] diff --git a/workbench-session-init/entrypoint/go.mod b/workbench-session-init/entrypoint/go.mod new file mode 100644 index 00000000..fff4e5d3 --- /dev/null +++ b/workbench-session-init/entrypoint/go.mod @@ -0,0 +1,9 @@ +module entrypoint + +go 1.22.2 + +require ( + github.com/otiai10/copy v1.14.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect +) diff --git a/workbench-session-init/entrypoint/go.sum b/workbench-session-init/entrypoint/go.sum new file mode 100644 index 00000000..21650d9b --- /dev/null +++ b/workbench-session-init/entrypoint/go.sum @@ -0,0 +1,6 @@ +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/workbench-session-init/entrypoint/main.go b/workbench-session-init/entrypoint/main.go new file mode 100644 index 00000000..5eb6afc0 --- /dev/null +++ b/workbench-session-init/entrypoint/main.go @@ -0,0 +1,170 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "time" + + cp "github.com/otiai10/copy" +) + +const ( + sourceDir = "/opt/session-components" + targetDir = "/mnt/init" +) + +var ( + // Read the PWB_SESSION_TYPE environment variable + sessionType = os.Getenv("PWB_SESSION_TYPE") + + // Set the copy options. + // Preserve permissions, times, and owner. + opt = cp.Options{ + PermissionControl: cp.PerservePermission, + PreserveTimes: true, + PreserveOwner: true, + NumOfWorkers: 20, + } + + // List of dependencies common to all session types + commonDeps = []string{ + "bin/git-credential-pwb", + "bin/focal", + "bin/jammy", + "bin/noble", + "bin/opensuse15", + "bin/postback", + "bin/pwb-supervisor", + "bin/quarto", + "bin/r-ldpath", + "bin/rhel8", + "bin/rhel9", + "bin/shared-run", + "R", + "resources", + "www", + "www-symbolmaps", + } + + // Map of session-specific dependencies + sessionDeps = map[string][]string{ + "jupyter": { + "bin/jupyter-session-run", + "bin/node", + "extras", + }, + "positron": { + "bin/positron-server", + "bin/positron-session-run", + "extras", + }, + "rstudio": { + "bin/node", + "bin/rsession-run", + }, + "vscode": { + "bin/pwb-code-server", + "bin/vscode-session-run", + "extras", + }, + } +) + +func main() { + if sessionType == "" { + fmt.Println("PWB_SESSION_TYPE environment variable is not set") + os.Exit(1) + } + + programStart := time.Now() + defer func() { + elapsed := time.Since(programStart) + fmt.Printf("Program took %s\n", elapsed) + }() + + filesToCopy, err := getFilesToCopy(sessionType) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = validateTargetDir(targetDir) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = copyFiles(sourceDir, targetDir, filesToCopy) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Println("Copy operation completed.") +} + +// getFilesToCopy returns the list of files to copy based on the session type. +func getFilesToCopy(sessionType string) ([]string, error) { + files := commonDeps + if deps, ok := sessionDeps[sessionType]; ok { + files = append(files, deps...) + } else { + return nil, fmt.Errorf("unknown session type: %s", sessionType) + } + return files, nil +} + +// validateTargetDir checks if the target directory exists and is empty. +func validateTargetDir(targetDir string) error { + if _, err := os.Stat(targetDir); os.IsNotExist(err) { + return fmt.Errorf("cannot find the copy target %s", targetDir) + } + + isEmpty, err := isDirEmpty(targetDir) + if err != nil { + return fmt.Errorf("error checking if target directory is empty: %v", err) + } + if !isEmpty { + return fmt.Errorf("target directory %s is not empty", targetDir) + } + + return nil +} + +// isDirEmpty checks if a directory is empty. +func isDirEmpty(dir string) (bool, error) { + f, err := os.Open(dir) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.ReadDir(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +// copyFiles copies the files from the source directory to the target directory. +// It uses the otiai10/copy package to copy files, with options to preserve +// permissions, times, and owner. +func copyFiles(src, dst string, filesToCopy []string) error { + fmt.Printf("Copying files from %s to %s\n", src, dst) + start := time.Now() + + for _, file := range filesToCopy { + srcPath := filepath.Join(src, file) + dstPath := filepath.Join(dst, file) + err := cp.Copy(srcPath, dstPath, opt) + if err != nil { + return fmt.Errorf("error copying %s: %v", srcPath, err) + } + } + + elapsed := time.Since(start) + fmt.Printf("Copy operation took %s\n", elapsed) + + return nil +} diff --git a/workbench-session-init/entrypoint/main_test.go b/workbench-session-init/entrypoint/main_test.go new file mode 100644 index 00000000..b16aa736 --- /dev/null +++ b/workbench-session-init/entrypoint/main_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "syscall" + "testing" +) + +func TestGetFilesToCopy(t *testing.T) { + tests := []struct { + sessionType string + expected []string + expectError bool + }{ + { + sessionType: "jupyter", + expected: append(commonDeps, sessionDeps["jupyter"]...), + expectError: false, + }, + { + sessionType: "positron", + expected: append(commonDeps, sessionDeps["positron"]...), + expectError: false, + }, + { + sessionType: "rstudio", + expected: append(commonDeps, sessionDeps["rstudio"]...), + expectError: false, + }, + { + sessionType: "vscode", + expected: append(commonDeps, sessionDeps["vscode"]...), + expectError: false, + }, + { + sessionType: "unknown", + expected: nil, + expectError: true, + }, + } + + for _, test := range tests { + t.Run(test.sessionType, func(t *testing.T) { + files, err := getFilesToCopy(test.sessionType) + if test.expectError { + if err == nil { + t.Errorf("Expected error for session type %s, but got none", test.sessionType) + } + } else { + if err != nil { + t.Errorf("Did not expect error for session type %s, but got: %v", test.sessionType, err) + } + if !reflect.DeepEqual(files, test.expected) { + t.Errorf("Files do not match for session type %s. Expected: %v, Got: %v", test.sessionType, test.expected, files) + } + } + }) + } +} + +func TestCopy(t *testing.T) { + // Create temporary source and destination directories + srcDir, err := os.MkdirTemp("", "src") + if err != nil { + t.Fatalf("Failed to create temporary source directory: %v", err) + } + defer os.RemoveAll(srcDir) + + dstDir, err := os.MkdirTemp("", "dst") + if err != nil { + t.Fatalf("Failed to create temporary destination directory: %v", err) + } + defer os.RemoveAll(dstDir) + + // Create a sample directory structure in the source directory that looks like: + // srcDir + // ├── file1.txt + // └── subdir1 + // ├── file2.txt + // └── subdir2 + // └── file3.txt + // |__ subdir3 + err = os.MkdirAll(filepath.Join(srcDir, "subdir1"), 0755) + if err != nil { + t.Fatalf("Failed to create subdir1: %v", err) + } + err = os.WriteFile(filepath.Join(srcDir, "file1.txt"), []byte("file1 content"), 0644) + if err != nil { + t.Fatalf("Failed to create file1.txt: %v", err) + } + err = os.WriteFile(filepath.Join(srcDir, "subdir1", "file2.txt"), []byte("file2 content"), 0600) + if err != nil { + t.Fatalf("Failed to create file2.txt: %v", err) + } + err = os.MkdirAll(filepath.Join(srcDir, "subdir1", "subdir2"), 0755) + if err != nil { + t.Fatalf("Failed to create subdir2: %v", err) + } + err = os.WriteFile(filepath.Join(srcDir, "subdir1", "subdir2", "file3.txt"), []byte("file3 content"), 0644) + if err != nil { + t.Fatalf("Failed to create file3.txt: %v", err) + } + err = os.MkdirAll(filepath.Join(srcDir, "subdir3"), 0755) + if err != nil { + t.Fatalf("Failed to create subdir3: %v", err) + } + + // Copy the directory structure from source to destination + // exclude subdir3 + filesToCopy := []string{ + "file1.txt", + "subdir1", + } + err = copyFiles(srcDir, dstDir, filesToCopy) + if err != nil { + t.Fatalf("Failed to copy files: %v", err) + } + + // Verify that the directory structure and files are correctly copied + verifyFile(t, filepath.Join(dstDir, "file1.txt"), 0644, os.Getuid(), os.Getgid()) + verifyFile(t, filepath.Join(dstDir, "subdir1", "file2.txt"), 0600, os.Getuid(), os.Getgid()) + verifyFile(t, filepath.Join(dstDir, "subdir1", "subdir2", "file3.txt"), 0644, os.Getuid(), os.Getgid()) + // Verify that subdir3 is not copied + if _, err := os.Stat(filepath.Join(dstDir, "subdir3")); !os.IsNotExist(err) { + t.Errorf("Directory subdir3 should not have been copied") + } +} + +func verifyFile(t *testing.T, path string, mode os.FileMode, uid, gid int) { + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Failed to stat file %s: %v", path, err) + } + + if info.Mode() != mode { + t.Errorf("File %s has incorrect permissions: got %v, want %v", path, info.Mode(), mode) + } + + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatalf("Failed to get file ownership for %s", path) + } + + if int(stat.Uid) != uid || int(stat.Gid) != gid { + t.Errorf("File %s has incorrect ownership: got %d:%d, want %d:%d", path, stat.Uid, stat.Gid, uid, gid) + } +} diff --git a/workbench-session-init/run.sh b/workbench-session-init/run.sh deleted file mode 100644 index 9c29238e..00000000 --- a/workbench-session-init/run.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -set -x - -S=/opt/session-components - -# The target should exist and be an empty directory. -T=/mnt/init - -if [ ! -d "${T}" ] ; then - echo "Cannot find the copy target ${T}" - exit 1 -fi - -echo "Copying files from /session-components to /mnt/init" -time cp -r $S/* $T diff --git a/workbench-session-init/test/goss.yaml b/workbench-session-init/test/goss.yaml index 7c9ab9e7..7b384ffd 100644 --- a/workbench-session-init/test/goss.yaml +++ b/workbench-session-init/test/goss.yaml @@ -7,7 +7,7 @@ file: exists: true mode: "0755" filetype: directory - /usr/local/bin/run.sh: + /usr/local/bin/entrypoint: exists: true filetype: file mode: "0755"