Merge pull request #1099 from github/3.7.2-patch

3.7.2 patch

Author: dooleydevin

.github/workflows/main.yml

@@ -14,9 +14,9 @@ jobs:
       run: |
         sudo apt-get update -y
         sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man
-        wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz"
-        tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz"
-        sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck
+        wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz"
+        tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz"
+        sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck
       if: matrix.os != 'macos-latest'
     - name: Install Dependencies (macOS)
       run: |

backup.config-example

@@ -90,3 +90,6 @@ GHE_NUM_SNAPSHOTS=10
 # When running an external mysql database, run this script to trigger a MySQL restore
 # rather than attempting to backup via backup-utils directly.
 #EXTERNAL_DATABASE_RESTORE_SCRIPT="/bin/false"
+
+# If set to 'yes', Pages data will be included in backup and restore. Defaults to 'yes'
+#GHE_BACKUP_PAGES=no

bin/ghe-backup

@@ -231,9 +231,14 @@ commands+=("
 echo \"Backing up Git repositories ...\"
 ghe-backup-repositories || printf %s \"repositories \" >> \"$failures_file\"")
 
-commands+=("
-echo \"Backing up GitHub Pages artifacts ...\"
-ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"")
+# Pages backups are skipped only if GHE_BACKUP_PAGES is explicitly set to 'no' to guarantee backward compatibility.
+# If a customer upgrades backup-utils but keeps the config file from a previous version, Pages backups still work as expected.
+
+if [ "$GHE_BACKUP_PAGES" != "no" ]; then
+  commands+=("
+  echo \"Backing up GitHub Pages artifacts ...\"
+  ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"")
+fi
 
 commands+=("
 echo \"Backing up storage data ...\"

bin/ghe-restore

@@ -447,9 +447,11 @@ commands+=("
 echo \"Restoring Gists ...\"
 ghe-restore-repositories-gist \"$GHE_HOSTNAME\"")
 
-commands+=("
-echo \"Restoring GitHub Pages artifacts ...\"
-ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3")
+if [ "$GHE_BACKUP_PAGES" != "no" ]; then
+  commands+=("
+  echo \"Restoring GitHub Pages artifacts ...\"
+  ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3")
+fi
 
 commands+=("
 echo \"Restoring SSH authorized keys ...\"

script/release

@@ -34,6 +34,9 @@ DEB_PKG_NAME = 'github-backup-utils'
 GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly?
 GH_STABLE_BRANCH = ""
 
+# If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI
+PUBLISH = ENV['PUBLISH'] == 'true' || false
+
 CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium
 
 <%- changes.each do |ch| -%>
@@ -332,7 +335,7 @@ def clean_up(version)
   `git pull --quiet origin #{GH_BASE_BRANCH} --prune`
   `git branch --quiet -D release-#{version} >/dev/null 2>&1`
   `git push --quiet origin :release-#{version} >/dev/null 2>&1`
-  `git branch --quiet -D tmp-packging >/dev/null 2>&1`
+  `git branch --quiet -D tmp-packaging >/dev/null 2>&1`
 end
 
 def is_base_branch_valid?(branch)
@@ -480,15 +483,21 @@ if $PROGRAM_NAME == __FILE__
     attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"]
     attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"]
 
-    puts 'Publishing release...'
-    publish_release res['id']
+    if PUBLISH
+      puts 'Publishing release...'
+      publish_release res['id']
+    end
 
     puts 'Cleaning up...'
     clean_up version
 
     puts "Updating #{GH_STABLE_BRANCH} branch..."
     update_stable_branch
 
+    if !PUBLISH
+      puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.'
+    end
+
     puts 'Released!'
   rescue RuntimeError => e
     $stderr.puts "Error: #{e}"

share/github-backup-utils/ghe-backup-es-rsync

@@ -4,6 +4,7 @@
 #/
 #/ Note: This command typically isn't called directly. It's invoked by
 #/ ghe-backup when the rsync strategy is used.
+# shellcheck disable=SC2086
 set -e
 
 # Bring in the backup configuration
@@ -53,15 +54,15 @@ ghe-rsync -avz \
 # Set up a trap to re-enable flushing on exit and remove temp file
 cleanup () {
   ghe_verbose "* Enabling ES index flushing ..."
-  echo '{"index":{"translog.disable_flush":false}}' |
+  echo '{"index":{"translog.flush_threshold_size":"512MB"}}' |
   ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
 }
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
 
 # Disable ES flushing and force a flush right now
 ghe_verbose "* Disabling ES index flushing ..."
-echo '{"index":{"translog.disable_flush":true}}' |
+echo '{"index":{"translog.flush_threshold_size":"1PB"}}' |
 ghe-ssh "$host" -- curl -s -XPUT  "localhost:9200/_settings" -d @- >/dev/null
 ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
 

test/test-ghe-host-check.sh

@@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions
   read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION)
   bu_major_minor="$bu_version_major.$bu_version_minor"
   releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json)
-  supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")')
+  latest_value=$(echo "$releases" | jq -r '.latest')
+  latest_major_version=$(echo $latest_value | cut -d "." -f 1-2)
+  # Replace "latest" with the derived major version in the releases string
+  releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g')
+  # Use the modified releases string as needed
+  supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")')
   # shellcheck disable=SC2207 # Command required as alternatives fail
-  keys=($(echo $releases | jq -r 'keys[]'))
+  keys=($(echo "$releases_with_replacement" | jq -r 'keys[]'))
 
   if [ -z "$supported" ]
   then
@@ -87,7 +92,6 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions
   GHE_TEST_REMOTE_VERSION=$bu_version_major.$bu_version_minor.999gm1 ghe-host-check
   ! GHE_TEST_REMOTE_VERSION=3.9999.1521793591.performancetest ghe-host-check
   GHE_TEST_REMOTE_VERSION=$((bu_version_major+1)).0.0 ghe-host-check
-
 )
 end_test
 

test/test-shellcheck.sh

@@ -11,8 +11,8 @@ begin_test "shellcheck: reports no errors or warnings"
   set -e
   # We manually install the latest Shellcheck on Linux builds as other options
   # are too old.
-  if [ -x "$BASE_PATH/shellcheck-latest/shellcheck" ]; then
-    shellcheck() { "$BASE_PATH/shellcheck-latest/shellcheck" "$@"; }
+  if [ -x "$BASE_PATH/shellcheck-stable/shellcheck" ]; then
+    shellcheck() { "$BASE_PATH/shellcheck-stable/shellcheck" "$@"; }
   fi
 
   if ! type shellcheck 1>/dev/null 2>&1; then