Merge branch 'master' into master

Author: bonsohi

bin/ghe-restore

@@ -86,6 +86,19 @@ while true; do
   esac
 done
 
+start_cron () {
+  echo "Starting cron ..."
+  if $CLUSTER; then
+    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron start"; then
+      echo "* Warning: Failed to start cron on one or more nodes"
+    fi
+  else
+    if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron start"; then
+      echo "* Warning: Failed to start cron"
+    fi
+  fi
+}
+
 cleanup () {
   if [ -n "$1" ]; then
     update_restore_status "$1"
@@ -103,6 +116,10 @@ cleanup () {
     fi
   fi
 
+  if ! $CRON_RUNNING; then
+    start_cron
+  fi
+
   # Cleanup SSH multiplexing
   ghe-ssh --clean
 }
@@ -249,6 +266,7 @@ update_restore_status () {
   fi
 }
 
+CRON_RUNNING=true
 # Update remote restore state file and setup failure trap
 trap "cleanup failed" EXIT
 update_restore_status "restoring"
@@ -271,9 +289,15 @@ if [ -d "$GHE_RESTORE_SNAPSHOT_PATH/mssql" ] || [ -d "$GHE_RESTORE_SNAPSHOT_PATH
     ac_db_ghe=$(echo 'ghe-mssql-console -y -n -q "SELECT name FROM sys.databases" | grep -i "ArtifactCache" | wc -l | tr -d " "' | ghe-ssh "$GHE_HOSTNAME" /bin/bash)
     ac_db_snapshot=$(find "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql/" -maxdepth 1 -name 'ArtifactCache*.bak' | wc -l | tr -d " ")
     if [[ $ac_db_ghe -gt 0  &&  $ac_db_snapshot -eq 0 ]]; then
-      echo "Error: $GHE_HOSTNAME contains ArtifactCache databases but no ArtifactCache databases are present in snapshot. Aborting" 1>&2
-      echo "Please delete ArtifactCache databases from $GHE_HOSTNAME and retry" 1>&2
-      echo "Steps to delete ArtifactCache databases can be found here: https://docs.github.com/en/enterprise-server@$RELEASE_VERSION/admin/github-actions/advanced-configuration-and-troubleshooting/deleting-artifact-cache-databases" 1>&2
+      echo "Error: $GHE_HOSTNAME has Actions Cache service enabled but no Actions Cache data is present in snapshot to restore. Aborting" 1>&2
+      echo "Please disable Actions cache service in $GHE_HOSTNAME and retry" 1>&2
+      echo "To disable Actions Cache service run as admin: ghe-actions-cache-disable" 1>&2
+      exit 1
+    fi
+    if [[ $ac_db_ghe -eq 0  &&  $ac_db_snapshot -gt 0 && ! $RESTORE_SETTINGS ]]; then
+      echo "Error: $GHE_HOSTNAME has Actions Cache service disabled but the snapshot is attempting to restore data for the service. Aborting" 1>&2
+      echo "Please enable Actions cache service in $GHE_HOSTNAME and retry" 1>&2
+      echo "To enable Actions Cache service run as admin: ghe-actions-cache-enable" 1>&2
       exit 1
     fi
   else
@@ -324,7 +348,7 @@ else
     fi
   fi
 fi
-
+CRON_RUNNING=false
 
 # Restore settings and license if restoring to an unconfigured appliance or when
 # specified manually.
@@ -492,16 +516,8 @@ if ! $RESTORE_SETTINGS; then
 fi
 
 # Start cron. Timerd will start automatically as part of the config run.
-echo "Starting cron ..."
-if $CLUSTER; then
-  if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron start"; then
-    echo "* Warning: Failed to start cron on one or more nodes"
-  fi
-else
-  if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron start"; then
-    echo "* Warning: Failed to start cron"
-  fi
-fi
+start_cron
+CRON_RUNNING=true
 
 # Clean up all stale replicas on configured instances.
 if ! $CLUSTER && $instance_configured; then

debian/changelog

@@ -1,58 +1,39 @@
-github-backup-utils (3.4.0) UNRELEASED; urgency=medium
-
-  * Add anchor to usage doc for settings restore #865
-
- -- Steve Culver <[email protected]>  Tue, 15 Feb 2022 19:25:09 +0000
-
-github-backup-utils (3.3.1) UNRELEASED; urgency=medium
-
-  * Fix compat issue with ghe-actions-start during maintenance mode #836
-
- -- [email protected]  Tue, 21 Dec 2021 23:38:01 +0000
-
-github-backup-utils (3.3.0) UNRELEASED; urgency=medium
-
-
- -- [email protected]  Wed, 08 Dec 2021 03:12:53 +0000
-
-github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+github-backup-utils (3.4.1) UNRELEASED; urgency=medium
 
+  * Simplify complex redirects for ghe-rsync #881
+  * On restore failure, restart cron on target host #883
 
- -- bonsohi@github.com  Wed, 08 Dec 2021 03:10:21 +0000
+ -- Donal Ellis <donal@github.com>  Fri, 22 Apr 2022 04:00:00 +0000
 
-github-backup-utils (3.3.0) UNRELEASED; urgency=medium
-
-
- -- [email protected]  Wed, 08 Dec 2021 02:56:47 +0000
-
-github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+github-backup-utils (3.3.2) UNRELEASED; urgency=medium
 
+  * Simplify complex redirects for ghe-rsync #881
+  * On restore failure, restart cron on target host #883
 
- -- bonsohi@github.com  Wed, 08 Dec 2021 02:51:06 +0000
+ -- Donal Ellis <donal@github.com>  Fri, 22 Apr 2022 00:53:45 +0000
 
-github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+github-backup-utils (3.4.0) UNRELEASED; urgency=medium
 
+  * Add anchor to usage doc for settings restore #865
 
- -- bonsohi@github.com  Wed, 08 Dec 2021 02:02:25 +0000
+ -- Steve Culver <steveculver@github.com>  Tue, 15 Feb 2022 19:25:09 +0000
 
-github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+github-backup-utils (3.3.1) UNRELEASED; urgency=medium
 
+  * Fix compat issue with ghe-actions-start during maintenance mode #836
 
- -- [email protected]  Wed, 08 Dec 2021 01:56:26 +0000
+ -- Balwinder Sohi <[email protected]>  Tue, 21 Dec 2021 23:38:01 +0000
 
 github-backup-utils (3.3.0) UNRELEASED; urgency=medium
 
-
- -- [email protected]  Wed, 08 Dec 2021 01:25:12 +0000
+ -- Balwinder Sohi <[email protected]>  Wed, 08 Dec 2021 03:12:53 +0000
 
 github-backup-utils (3.3.0.rc1) UNRELEASED; urgency=medium
 
-
  -- Nick Iodice <[email protected]>  Tue, 09 Nov 2021 19:56:08 +0000
 
 github-backup-utils (3.2.0) UNRELEASED; urgency=medium
 
-
  -- Brett Westover <[email protected]>  Tue, 28 Sep 2021 16:50:00 +0000
 
 github-backup-utils (3.2.0.rc3) UNRELEASED; urgency=medium
@@ -86,7 +67,6 @@ github-backup-utils (3.2.0) UNRELEASED; urgency=medium
 
 github-backup-utils (3.1.0) UNRELEASED; urgency=medium
 
-
  -- Zachary Mark <[email protected]>  Thu, 03 Jun 2021 16:55:16 +0000
 
 github-backup-utils (3.1.0~rc1) UNRELEASED; urgency=medium

share/github-backup-utils/ghe-backup-settings

@@ -116,6 +116,16 @@ if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then
   backup-secret "Actions Launch service private key" "actions-launch-app-app-private-key" "secrets.launch.azp-app-private-key"
 fi
 
+if ghe-ssh "$host" -- ghe-config --true app.packages.enabled; then
+  backup-secret "Packages aws access key" "packages-aws-access-key" "secrets.packages.aws-access-key"
+  backup-secret "Packages aws secret key" "packages-aws-secret-key" "secrets.packages.aws-secret-key"
+  backup-secret "Packages s3 bucket" "packages-s3-bucket" "secrets.packages.s3-bucket"
+  backup-secret "Packages storage service url" "packages-service-url" "secrets.packages.service-url"
+  backup-secret "Packages blob storage type" "packages-blob-storage-type" "secrets.packages.blob-storage-type"
+  backup-secret "Packages azure connection string" "packages-azure-connection-string" "secrets.packages.azure-connection-string"
+  backup-secret "Packages azure container name" "packages-azure-container-name" "secrets.packages.azure-container-name"
+fi
+
 if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/idp.crt"; then
   echo "* Transferring SAML keys ..." 1>&3
   ghe-ssh $host -- sudo tar -C $GHE_REMOTE_DATA_USER_DIR/common/ -cf - "idp.crt saml-sp.p12" > saml-keys.tar

share/github-backup-utils/ghe-restore-packages

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-restore-packages <host>
+#/
+#/ Note: This script typically isn't called directly. It's invoked by the
+#/ ghe-restore command.
+set -e
+
+# Bring in the backup configuration
+# shellcheck source=share/github-backup-utils/ghe-backup-config
+. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
+
+# Show usage and bail with no arguments
+[ -z "$*" ] && print_usage
+
+bm_start "$(basename $0)"
+
+# Grab host arg
+GHE_HOSTNAME="$1"
+
+# The snapshot to restore should be set by the ghe-restore command but this lets
+# us run this script directly.
+: ${GHE_RESTORE_SNAPSHOT:=current}
+
+# Path to snapshot dir we're restoring from
+export GHE_RESTORE_SNAPSHOT_PATH="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
+
+port=$(ssh_port_part "$GHE_HOSTNAME")
+export port
+host=$(ssh_host_part "$GHE_HOSTNAME")
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "$host"
+
+# Restore Packages settings.
+ghe_verbose "Restoring Packages settings ..."
+
+restore-secret "Packages aws access key" "packages-aws-access-key" "secrets.packages.aws-access-key"
+restore-secret "Packages aws secret key" "packages-aws-secret-key" "secrets.packages.aws-secret-key"
+restore-secret "Packages s3 bucket" "packages-s3-bucket" "secrets.packages.s3-bucket"
+restore-secret "Packages storage service url" "packages-service-url" "secrets.packages.service-url"
+restore-secret "Packages blob storage type" "packages-blob-storage-type" "secrets.packages.blob-storage-type"
+restore-secret "Packages azure connection string" "packages-azure-connection-string" "secrets.packages.azure-connection-string"
+restore-secret "Packages azure container name" "packages-azure-container-name" "secrets.packages.azure-container-name"
+
+bm_end "$(basename $0)"

share/github-backup-utils/ghe-restore-settings

@@ -33,6 +33,9 @@ echo "Restoring settings and applying configuration ..."
 # Restore external MySQL password if running external MySQL DB.
 restore-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql"
 
+echo "Restoring packages settings ..."
+ghe-restore-packages "$GHE_HOSTNAME" 1>&3
+
 # work around issue importing settings with bad storage mode values
 ( cat "$GHE_RESTORE_SNAPSHOT_PATH/settings.json" && echo ) |
   sed 's/"storage_mode": "device"/"storage_mode": "rootfs"/' |

share/github-backup-utils/ghe-rsync

@@ -11,11 +11,6 @@ set -o pipefail
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
 
-# Filter vanished file warnings from both stdout (rsync versions < 3.x) and
-# stderr (rsync versions >= 3.x). The complex redirections are necessary to
-# filter stderr while also keeping stdout and stderr separated.
-ignoreout='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
-
 # Check for --ignore-missing-args parameter support and remove if unavailable.
 if rsync -h | grep '\-\-ignore-missing-args' >/dev/null 2>&1; then
   parameters=("$@")
@@ -25,9 +20,15 @@ else
   done
 fi
 
-(rsync "${parameters[@]}" $GHE_EXTRA_RSYNC_OPTS 3>&1 1>&2 2>&3 3>&- |
-  (egrep -v "$ignoreout" || true)) 3>&1 1>&2 2>&3 3>&- |
-  (egrep -v "$ignoreout" || true)
+ignoreout='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
+rsync_version_check=`rsync --version | egrep "version 3.[0-9]*.[0-9]*"`
+if [ ! -z "$rsync_version_check" ]; then
+  # rsync >= 3.x sends errors to stderr. so, we need to redirect to stdout before the pipe
+  rsync "${parameters[@]}" $GHE_EXTRA_RSYNC_OPTS 2>&1 | (egrep -v "$ignoreout" || true)
+else
+  # rsync <3.x sends errors to stdout.
+  rsync "${parameters[@]}" $GHE_EXTRA_RSYNC_OPTS | (egrep -v "$ignoreout" || true)
+fi
 res=$?
 
 # Suppress exits with 24.

share/github-backup-utils/version

@@ -1 +1 @@
-3.4.0
+3.4.1