ci: integrated scripts/choco-install.ps1 and scripts/brew-install.sh and all jobs use nix-shell --run

Author: CMCDragonkai

.gitlab-ci.yml

@@ -5,27 +5,33 @@ workflow:
       when: never
     - when: always
 
-default:
-  interruptible: true
-
 variables:
+  GIT_SUBMODULE_STRATEGY: recursive
   GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
   GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
-  GIT_SUBMODULE_STRATEGY: recursive
   # Cache .npm
   NPM_CONFIG_CACHE: "${CI_PROJECT_DIR}/tmp/npm"
   # Prefer offline node module installation
   NPM_CONFIG_PREFER_OFFLINE: "true"
   # Homebrew cache only used by macos runner
   HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew"
 
+default:
+  interruptible: true
+  before_script:
+    # Replace this in windows runners that use powershell
+    # with `mkdir -Force "$CI_PROJECT_DIR/tmp"`
+    - mkdir -p "$CI_PROJECT_DIR/tmp"
+
 # Cached directories shared between jobs & pipelines per-branch per-runner
 cache:
   key: $CI_COMMIT_REF_SLUG
   paths:
     - ./tmp/npm/
     # Homebrew cache is only used by the macos runner
     - ./tmp/Homebrew
+    # Chocolatey cache is only used by the windows runner
+    - ./tmp/chocolatey/
     # `jest` cache is configured in jest.config.js
     - ./tmp/jest/
 
@@ -87,20 +93,20 @@ build:merge:
     # Required for `gh pr create`
     - git remote add upstream "$GH_PROJECT_URL"
     - >
-        nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
-        gh pr create \
-          --head staging \
-          --base master \
-          --title "ci: merge staging to master" \
-          --body "This is an automatic PR generated by the pipeline CI/CD. This will be automatically fast-forward merged if successful." \
-          --assignee "@me" \
-          --no-maintainer-edit \
-          --repo "$GH_PROJECT_PATH" || true;
-        printf "Pipeline Attempt on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \
-        | gh pr comment staging \
-          --body-file - \
-          --repo "$GH_PROJECT_PATH";
-        '
+      nix-shell --run '
+      gh pr create \
+        --head staging \
+        --base master \
+        --title "ci: merge staging to master" \
+        --body "This is an automatic PR generated by the pipeline CI/CD. This will be automatically fast-forward merged if successful." \
+        --assignee "@me" \
+        --no-maintainer-edit \
+        --repo "$GH_PROJECT_PATH" || true;
+      printf "Pipeline Attempt on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \
+      | gh pr comment staging \
+        --body-file - \
+        --repo "$GH_PROJECT_PATH";
+      '
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
@@ -112,10 +118,10 @@ build:linux:
   needs: []
   script:
     - >
-        nix-shell --run '
-        npm run build --verbose;
-        npm test -- --ci --coverage;
-        '
+      nix-shell --run '
+      npm run build --verbose;
+      npm test -- --ci --coverage;
+      '
   artifacts:
     when: always
     reports:
@@ -140,10 +146,10 @@ build:windows:
   tags:
     - windows
   before_script:
-    - choco install nodejs --version=16.14.2 -y
-    - refreshenv
+    - mkdir -Force "$CI_PROJECT_DIR/tmp"
   script:
-    - npm config set msvs_version 2019
+    - .\scripts\choco-install.ps1
+    - refreshenv
     - npm install --ignore-scripts
     - $env:Path = "$(npm bin);" + $env:Path
     - npm run build --verbose
@@ -165,15 +171,10 @@ build:macos:
   tags:
     - shared-macos-amd64
   image: macos-11-xcode-12
-  variables:
-    HOMEBREW_NO_INSTALL_UPGRADE: "true"
-    HOMEBREW_NO_INSTALL_CLEANUP: "true"
-  before_script:
+  script:
     - eval "$(brew shellenv)"
-    - brew install node@16
-    - brew link --overwrite node@16
+    - ./scripts/brew-install.sh
     - hash -r
-  script:
     - npm install --ignore-scripts
     - export PATH="$(npm bin):$PATH"
     - npm run build --verbose
@@ -197,14 +198,13 @@ build:prerelease:
     - build:macos
   # Don't interrupt publishing job
   interruptible: false
-  before_script:
-    - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc
   script:
+    - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc
     - echo 'Publishing library prerelease'
     - >
-        nix-shell --run '
-        npm publish --tag prerelease --access public;
-        '
+      nix-shell --run '
+      npm publish --tag prerelease --access public;
+      '
   after_script:
     - rm -f ./.npmrc
   rules:
@@ -234,12 +234,12 @@ integration:merge:
     GIT_DEPTH: 0
   script:
     - >
-        nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
-        printf "Pipeline Succeeded on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \
-        | gh pr comment staging \
-          --body-file - \
-          --repo "$GH_PROJECT_PATH";
-        '
+      nix-shell --run '
+      printf "Pipeline Succeeded on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \
+      | gh pr comment staging \
+        --body-file - \
+        --repo "$GH_PROJECT_PATH";
+      '
     - git remote add upstream "$GH_PROJECT_URL"
     - git checkout origin/master
     # Merge up to the current commit (not the latest commit)
@@ -260,14 +260,13 @@ release:distribution:
     - integration:merge
   # Don't interrupt publishing job
   interruptible: false
-  before_script:
-    - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc
   script:
+    - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc
     - echo 'Publishing library'
     - >
-        nix-shell --run '
-        npm publish --access public;
-        '
+      nix-shell --run '
+      npm publish --access public;
+      '
   after_script:
     - rm -f ./.npmrc
   rules:

scripts/brew-install.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
+
+export HOMEBREW_NO_INSTALL_UPGRADE=1
+export HOMEBREW_NO_INSTALL_CLEANUP=1
+export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
+export HOMEBREW_NO_AUTO_UPDATE=1
+export HOMEBREW_NO_ANALYTICS=1
+
+brew install node@16
+brew link --overwrite node@16

scripts/choco-install.ps1

@@ -0,0 +1,30 @@
+function Save-ChocoPackage {
+  param (
+      $PackageName
+  )
+  Rename-Item -Path "$env:ChocolateyInstall\lib\$PackageName\$PackageName.nupkg" -NewName "$PackageName.nupkg.zip" -ErrorAction:SilentlyContinue
+  Expand-Archive -LiteralPath "$env:ChocolateyInstall\lib\$PackageName\$PackageName.nupkg.zip" -DestinationPath "$env:ChocolateyInstall\lib\$PackageName" -Force
+  Remove-Item "$env:ChocolateyInstall\lib\$PackageName\_rels" -Recurse
+  Remove-Item "$env:ChocolateyInstall\lib\$PackageName\package" -Recurse
+  Remove-Item "$env:ChocolateyInstall\lib\$PackageName\[Content_Types].xml"
+  New-Item -Path "${PSScriptRoot}\..\tmp\chocolatey\$PackageName" -ItemType "directory" -ErrorAction:SilentlyContinue
+  choco pack "$env:ChocolateyInstall\lib\$PackageName\$PackageName.nuspec" --outdir "${PSScriptRoot}\..\tmp\chocolatey\$PackageName"
+}
+
+# Check for existence of required environment variables
+if ( $null -eq $env:ChocolateyInstall ) {
+  [Console]::Error.WriteLine('Missing $env:ChocolateyInstall environment variable')
+  exit 1
+}
+
+# Add the cached packages with source priority 1 (Chocolatey community is 0)
+New-Item -Path "${PSScriptRoot}\..\tmp\chocolatey" -ItemType "directory" -ErrorAction:SilentlyContinue
+choco source add --name="cache" --source="${PSScriptRoot}\..\tmp\chocolatey" --priority=1
+
+# Install nodejs v16.15.1 (will use cache if exists)
+$nodejs = "nodejs.install"
+choco install "$nodejs" --version="16.15.1" --require-checksums -y
+# Internalise nodejs to cache if doesn't exist
+if ( -not (Test-Path -Path "${PSScriptRoot}\..\tmp\chocolatey\$nodejs\$nodejs.16.15.1.nupkg" -PathType Leaf) ) {
+  Save-ChocoPackage -PackageName $nodejs
+}

shell.nix

@@ -4,6 +4,7 @@ with pkgs;
 mkShell {
   nativeBuildInputs = [
     nodejs
+    gitAndTools.gh
   ];
   shellHook = ''
     echo "Entering $(npm pkg get name)"