Merge pull request #114 from github/lildude/merge-update

Merge upstream

Author: lildude

bin/ghe-restore

@@ -109,7 +109,7 @@ if $cluster; then
   snapshot_instance_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version)
   if ! echo $snapshot_instance_version | \
     grep -Eq "v2\.[5-9]|v2\.[1-9][0-9]|v[3-9]|v[1-9][0-9]"; then
-    echo "Error: Snapshot must be from GitHub Enterprise v2.5.0 or above to be restored"
+    echo "Error: Snapshot must be from GitHub Enterprise v2.5.0 or above to be restored" >&2
     echo "       into a cluster (detected $snapshot_instance_version).  Aborting." >&2
     exit 1
   fi
@@ -123,6 +123,21 @@ if ghe-ssh "$GHE_HOSTNAME" -- \
   exit 1
 fi
 
+# Only allow restores of 2.9 and 2.10 snapshots that have run the audit log migration to 2.11 and above
+if ! $force; then
+  snapshot_instance_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version)
+  snapshot_version_major=$(echo "${snapshot_instance_version#v}" | cut -f 1 -d .)
+  snapshot_version_minor=$(echo "$snapshot_instance_version"     | cut -f 2 -d .)
+  if ! test -f $GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete && \
+     [ "$snapshot_version_major" -eq 2 ] && [ "$snapshot_version_minor" -lt 11 ] && \
+     [ "$GHE_VERSION_MAJOR" -eq 2 ] && [ "$GHE_VERSION_MINOR" -ge 11 ]; then
+        echo "Error: Snapshot must be from GitHub Enterprise v2.9 or v2.10 after running the" >&2
+        echo "       audit log migration, or from v2.11.0 or above." >&2
+        echo "Please see https://git.io/v5rCE for the audit log migration procedure." >&2
+        exit 1
+  fi
+fi
+
 # Prompt to verify the restore host given is correct. Restoring overwrites
 # important data on the destination appliance that cannot be recovered. This is
 # mostly to prevent accidents where the backup host is given to restore instead
@@ -339,6 +354,11 @@ else
   ghe-restore-es-${GHE_BACKUP_STRATEGY} "$GHE_HOSTNAME" 1>&3
 fi
 
+# Restore the audit log migration sentinel file, if it exists in the snapshot
+if test -f $GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete; then
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"
+fi
+
 # Restart an already running memcached to reset the cache after restore
 if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
     echo "Restarting memcached ..." 1>&3

debian/changelog

@@ -1,3 +1,12 @@
+github-backup-utils (2.11.0) UNRELEASED; urgency=medium
+
+  * Use calculated routes when backing up storage data from a cluster #318
+  * Add SSH multiplexing support #321
+  * Optimise route generation and finalisation during cluster restores #322
+  * Prefer the SSH port specified on the command line #324
+
+ -- Colin Seymour <[email protected]>  Wed, 13 Sep 2017 16:31:20 +0000
+
 github-backup-utils (2.10.0) UNRELEASED; urgency=medium
 
   * Include the user data directory in the benchmark name #311

script/cibuild

@@ -11,6 +11,7 @@ REMOTE_VERSIONS="
   2.0.0
   2.2.0
   2.5.0
+  2.11.0
 "
 
 # Enable verbose logging of ssh commands

share/github-backup-utils/ghe-backup-es-audit-log

@@ -31,6 +31,13 @@ if ! indices=$(ghe-ssh "$host" "curl -s \"localhost:$es_port/_cat/indices/audit_
   exit 1
 fi
 
+# Determine if the audit log migration has occurred or is needed.
+if echo 'set -o pipefail; ! test -e /data/user/common/es-scan-complete && test -f /usr/local/share/enterprise/run-audit-log-transitions.sh' | ghe-ssh "$host" /bin/bash; then
+  if echo 'set -o pipefail; echo n | /usr/local/share/enterprise/run-audit-log-transitions.sh > /dev/null 2>&1 && touch /data/user/common/es-scan-complete' | ghe-ssh "$host" /bin/bash; then
+    touch $GHE_SNAPSHOT_DIR/es-scan-complete
+  fi
+fi
+
 current_index=audit_log-$(ghe-ssh "$host" 'date +"%Y-%m"')
 
 for index in $indices; do

share/github-backup-utils/ghe-backup-es-rsync

@@ -96,4 +96,9 @@ ghe-rsync -avz \
     "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
     "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
 
+# "Backup" audit log migration sentinel file
+if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"; then
+  touch $GHE_SNAPSHOT_DIR/es-scan-complete
+fi
+
 bm_end "$(basename $0)"

share/github-backup-utils/ghe-ssh

@@ -58,7 +58,7 @@ if echo "$*" | grep "[|;]" >/dev/null || [ $(echo "$*" | wc -l) -gt 1 ]; then
 fi
 
 if [ -z "$GHE_DISABLE_SSH_MUX" ]; then
-  controlpath="$TMPDIR/.ghe-sshmux-$(echo -n "$user@$host:$port" | sha256sum | cut -c 1-8)"
+  controlpath="$TMPDIR/.ghe-sshmux-$(echo -n "$user@$host:$port" | git hash-object --stdin | cut -c 1-8)"
   opts="-o ControlMaster=auto -o ControlPath=\"$controlpath\" -o ControlPersist=10m -o ServerAliveInterval=10 $opts"
   # Workaround for https://bugzilla.mindrot.org/show_bug.cgi?id=1988
   [ -S $controlpath ] || ssh -f -p $port $opts -o BatchMode=yes "$host" -- /bin/true 1>/dev/null 2>&1 || true

share/github-backup-utils/ghe-ssh-config

@@ -25,7 +25,7 @@ proxy_user="${proxy_host%@*}"
 
 opts="$GHE_EXTRA_SSH_OPTS"
 
-[ -z "$GHE_DISABLE_SSH_MUX" ] && opts="-o ControlMaster=auto -o ControlPath=\"$TMPDIR/.ghe-sshmux-$(echo -n "$proxy_user@$proxy_host:$proxy_port" | sha256sum | cut -c 1-8)\" -o ControlPersist=10m -o ServerAliveInterval=10 $opts"
+[ -z "$GHE_DISABLE_SSH_MUX" ] && opts="-o ControlMaster=auto -o ControlPath=\"$TMPDIR/.ghe-sshmux-$(echo -n "$proxy_user@$proxy_host:$proxy_port" | git hash-object --stdin | cut -c 1-8)\" -o ControlPersist=10m -o ServerAliveInterval=10 $opts"
 
 for host in $hosts; do
   cat <<EOF

test/test-ghe-backup.sh

@@ -49,6 +49,9 @@ if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
 
     # Create a fake UUID
     echo "fake uuid" > "$GHE_REMOTE_DATA_USER_DIR/common/uuid"
+
+    # Create fake audit log migration sentinel file
+    touch "$GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"
 fi
 
 # Create some fake elasticsearch data in the remote data directory
@@ -154,6 +157,9 @@ begin_test "ghe-backup first snapshot"
 
         # check that ca certificates were backed up
         [ "$(cat "$GHE_DATA_DIR/current/ssl-ca-certificates.tar")" = "fake ghe-export-ssl-ca-certificates data" ]
+
+        # verify the audit log migration sentinel file has been created
+        [ -f "$GHE_DATA_DIR/current/es-scan-complete" ]
     fi
 
     # verify that ghe-backup wrote its version information to the host
@@ -241,6 +247,9 @@ begin_test "ghe-backup subsequent snapshot"
 
         # check that ca certificates were backed up
         [ "$(cat "$GHE_DATA_DIR/current/ssl-ca-certificates.tar")" = "fake ghe-export-ssl-ca-certificates data" ]
+
+        # verify the audit log migration sentinel file has been created
+        [ -f "$GHE_DATA_DIR/current/es-scan-complete" ]
     fi
 )
 end_test
@@ -344,6 +353,9 @@ begin_test "ghe-backup with relative data dir path"
 
         # check that ca certificates were backed up
         [ "$(cat "$GHE_DATA_DIR/current/ssl-ca-certificates.tar")" = "fake ghe-export-ssl-ca-certificates data" ]
+
+        # verify the audit log migration sentinel file has been created
+        [ -f "$GHE_DATA_DIR/current/es-scan-complete" ]
     fi
 
     # verify that ghe-backup wrote its version information to the host

test/test-ghe-restore.sh

@@ -83,6 +83,10 @@ echo "fake ghe-export-ssl-ca-certificates data" > "$GHE_DATA_DIR/current/ssl-ca-
 echo "fake license data" > "$GHE_DATA_DIR/current/enterprise.ghl"
 echo "fake manage password hash data" > "$GHE_DATA_DIR/current/manage-password"
 echo "rsync" > "$GHE_DATA_DIR/current/strategy"
+echo "$GHE_REMOTE_VERSION" >  "$GHE_DATA_DIR/current/version"
+if [ "$GHE_VERSION_MAJOR" -eq 2 ]; then
+  touch "$GHE_DATA_DIR/current/es-scan-complete"
+fi
 
 begin_test "ghe-restore into configured vm"
 (
@@ -152,6 +156,11 @@ begin_test "ghe-restore into configured vm"
 
         # verify the UUID was transferred
         diff -ru "$GHE_DATA_DIR/current/uuid" "$GHE_REMOTE_DATA_USER_DIR/common/uuid"
+
+        # verify the audit log migration sentinel file has been created on 2.9 and above
+        if [ "$GHE_VERSION_MAJOR" -eq 2 ] && [ "$GHE_VERSION_MINOR" -ge 9 ]; then
+          [ -f "$GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete" ]
+        fi
     fi
 )
 end_test
@@ -292,6 +301,11 @@ begin_test "ghe-restore -c into unconfigured vm"
 
         # verify ghe-export-ssl-ca-certificates was run
         grep -q "fake ghe-export-ssl-ca-certificates data" "$TRASHDIR/restore-out"
+
+        # verify the audit log migration sentinel file has been created on 2.9 and above
+        if [ "$GHE_VERSION_MAJOR" -eq 2 ] && [ "$GHE_VERSION_MINOR" -ge 9 ]; then
+          [ -f "$GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete" ]
+        fi
     fi
 )
 end_test
@@ -365,6 +379,11 @@ begin_test "ghe-restore into unconfigured vm"
 
         # verify no config run after restore on unconfigured instance
         ! grep -q "ghe-config-apply OK" "$TRASHDIR/restore-out"
+
+        # verify the audit log migration sentinel file has been created on 2.9 and above
+        if [ "$GHE_VERSION_MAJOR" -eq 2 ] && [ "$GHE_VERSION_MINOR" -ge 9 ]; then
+          [ -f "$GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete" ]
+        fi
     fi
 )
 end_test
@@ -418,6 +437,11 @@ begin_test "ghe-restore with host arg"
 
         # verify the UUID was transferred
         diff -ru "$GHE_DATA_DIR/current/uuid" "$GHE_REMOTE_DATA_USER_DIR/common/uuid"
+
+        # verify the audit log migration sentinel file has been created on 2.9 and above
+        if [ "$GHE_VERSION_MAJOR" -eq 2 ] && [ "$GHE_VERSION_MINOR" -ge 9 ]; then
+          [ -f "$GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete" ]
+        fi
     fi
 )
 end_test
@@ -628,3 +652,55 @@ begin_test "ghe-restore fails when restore to an active HA pair"
     echo $output | grep -q "Error: Restoring to an appliance with replication enabled is not supported."
 )
 end_test
+
+begin_test "ghe-restore fails when restore 2.9/2.10 snapshot without audit log migration sentinel file to 2.11"
+(
+  set -e
+
+  # noop if not testing against 2.11
+  if [ "$GHE_VERSION_MAJOR" -le 1 ] || [ "$GHE_VERSION_MINOR" -ne 11 ]; then
+    exit 0
+  fi
+
+  rm -rf "$GHE_REMOTE_ROOT_DIR"
+  setup_remote_metadata
+
+  echo "rsync" > "$GHE_DATA_DIR/current/strategy"
+  echo "v2.9.10" > "$GHE_DATA_DIR/current/version"
+  rm "$GHE_DATA_DIR/current/es-scan-complete"
+
+  ! output=$(ghe-restore -v localhost 2>&1)
+
+  echo $output | grep -q "Error: Snapshot must be from GitHub Enterprise v2.9 or v2.10 after running the"
+
+  echo "v2.10.5" > "$GHE_DATA_DIR/current/version"
+  ! output=$(ghe-restore -v localhost 2>&1)
+
+  echo $output | grep -q "Error: Snapshot must be from GitHub Enterprise v2.9 or v2.10 after running the"
+)
+end_test
+
+begin_test "ghe-restore force restore of 2.9/2.10 snapshot without audit log migration sentinel file to 2.11"
+(
+  set -e
+
+  # noop if not testing against 2.11
+  if [ "$GHE_VERSION_MAJOR" -le 1 ] || [ "$GHE_VERSION_MINOR" -ne 11 ]; then
+    exit 0
+  fi
+
+  rm -rf "$GHE_REMOTE_ROOT_DIR"
+  setup_remote_metadata
+
+  echo "rsync" > "$GHE_DATA_DIR/current/strategy"
+  echo "v2.9.10" > "$GHE_DATA_DIR/current/version"
+
+  # Create fake remote repositories dir
+  mkdir -p "$GHE_REMOTE_DATA_USER_DIR/repositories"
+
+  ghe-restore -v -f localhost
+
+  echo "v2.10.5" > "$GHE_DATA_DIR/current/version"
+  ghe-restore -v -f localhost
+)
+end_test