Merge pull request #167 from github/byodb-to-non-byodb

cainejette · web-flow · commit c102adb3324d · 2020-07-01T13:54:27.000-07:00
Add checks for non-BYODB to BYODB restores, and vice versa.
diff --git a/bin/ghe-restore b/bin/ghe-restore
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-#/ Usage: ghe-restore [-fchv] [--version] [-s <snapshot-id>] [<host>]
+#/ Usage: ghe-restore [-cfhv] [--version] [--skip-mysql] [-s <snapshot-id>] [<host>]
 #/
 #/ Restores a GitHub instance from local backup snapshots.
 #/
@@ -9,40 +9,54 @@
 #/ <https://enterprise.github.com/help/articles/adding-an-ssh-key-for-shell-access>
 #/
 #/ OPTIONS:
-#/   -f | --force      Don't prompt for confirmation before restoring.
-#/   -c | --config     Restore appliance settings and license in addition to
-#/                     datastores. Settings are not restored by default to
-#/                     prevent overwriting different configuration on the
-#/                     restore host.
-#/   -v | --verbose    Enable verbose output.
-#/   -h | --help       Show this message.
-#/        --version    Display version information and exit.
-#/   -s <snapshot-id>  Restore from the snapshot with the given id. Available
-#/                     snapshots may be listed under the data directory.
-#/   <host>            The <host> is the hostname or IP of the GitHub Enterprise
-#/                     instance. The <host> may be omitted when the
-#/                     GHE_RESTORE_HOST config variable is set in backup.config.
-#/                     When a <host> argument is provided, it always overrides
-#/                     the configured restore host.
+#/   -c | --config       Restore appliance settings and license in addition to
+#/                       datastores. Settings are not restored by default to
+#/                       prevent overwriting different configuration on the
+#/                       restore host.
+#/   -f | --force        Don't prompt for confirmation before restoring.
+#/   -h | --help         Show this message.
+#/   -v | --verbose      Enable verbose output.
+#/        --skip-mysql   Skip MySQL restore steps. Only applicable to external databases.
+#/        --version      Display version information and exit.
+#/
+#/   -s <snapshot-id>    Restore from the snapshot with the given id. Available
+#/                       snapshots may be listed under the data directory.
+#/
+#/   <host>              The <host> is the hostname or IP of the GitHub Enterprise
+#/                       instance. The <host> may be omitted when the
+#/                       GHE_RESTORE_HOST config variable is set in backup.config.
+#/                       When a <host> argument is provided, it always overrides
+#/                       the configured restore host.
 #/
 
 set -e
 
 # Parse arguments
-restore_settings=false
-force=false
+: ${RESTORE_SETTINGS:=false}
+export RESTORE_SETTINGS
+
+: ${FORCE:=false}
+export FORCE
+
+: ${SKIP_MYSQL:=false}
+export SKIP_MYSQL
+
 while true; do
   case "$1" in
+    --skip-mysql)
+      SKIP_MYSQL=true
+      shift
+      ;;
     -f|--force)
-      force=true
+      FORCE=true
       shift
       ;;
     -s)
       snapshot_id="$(basename "$2")"
       shift 2
       ;;
     -c|--config)
-      restore_settings=true
+      RESTORE_SETTINGS=true
       shift
       ;;
     -h|--help)
@@ -116,10 +130,10 @@ ghe_remote_version_required "$GHE_HOSTNAME"
 
 # Figure out if this instance has been configured or is entirely new.
 instance_configured=false
-if ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/configured' ]"; then
+if is_instance_configured; then
   instance_configured=true
 else
-  restore_settings=true
+  RESTORE_SETTINGS=true
 fi
 
 # Figure out if we're restoring into cluster
@@ -137,6 +151,11 @@ if ! $CLUSTER && [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
   exit 1
 fi
 
+# Ensure target appliance and restore snapshot are a compatible combination with respect to BYODB
+if ! ghe-restore-external-database-compatibility-check; then
+  exit 1
+fi
+
 # Figure out if this appliance is in a replication pair
 if ghe-ssh "$GHE_HOSTNAME" -- \
   "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then
@@ -149,7 +168,7 @@ fi
 # important data on the destination appliance that cannot be recovered. This is
 # mostly to prevent accidents where the backup host is given to restore instead
 # of a separate restore host since they're used in such close proximity.
-if $instance_configured && ! $force; then
+if $instance_configured && ! $FORCE; then
   echo
   echo "WARNING: All data on GitHub Enterprise appliance $hostname ($GHE_REMOTE_VERSION)"
   echo "         will be overwritten with data from snapshot ${GHE_RESTORE_SNAPSHOT}."
@@ -234,7 +253,7 @@ fi
 
 # Restore settings and license if restoring to an unconfigured appliance or when
 # specified manually.
-if $restore_settings; then
+if $RESTORE_SETTINGS; then
   ghe-restore-settings "$GHE_HOSTNAME"
 fi
 
@@ -255,10 +274,10 @@ if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft"
   fi
 
-if is_service_external 'mysql' "$GHE_RESTORE_SNAPSHOT_PATH/settings.json"; then
+if is_external_database_snapshot; then
    appliance_strategy="external"
    backup_snapshot_strategy="external"
-else 
+else
   if is_binary_backup_feature_on; then
     appliance_strategy="binary"
   else
@@ -272,8 +291,12 @@ else
   fi
 fi
 
-echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
-ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
+if is_external_database_target_or_snapshot && $SKIP_MYSQL; then
+  echo "Skipping MySQL restore."
+else
+  echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
+  ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
+fi
 
 commands=("
 echo \"Restoring Redis database ...\"
diff --git a/docs/usage.md b/docs/usage.md
@@ -16,6 +16,8 @@ You can supply your own configuration file or use the example configuration file
 
 An example configuration file with documentation on possible settings can found in [backup.config-example](../backup.config-example).
 
+There are a number of command line options that can also be passed to the `ghe-restore` command. Of particular note, if you use an external MySQL service but are restoring from a snapshot prior to enabling this, or vice versa, you must migrate the MySQL data outside of the context of backup-utils first, then pass the `--skip-mysql` flag to `ghe-restore`.
+
 ## Example backup and restore usage
 
 The following assumes that `GHE_HOSTNAME` is set to "github.example.com" in
diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config
@@ -392,6 +392,9 @@ is_binary_backup(){
   test -f "$1/mysql-binary-backup-sentinel"
 }
 
+# Check if a given service is managed externally on the appliance or in a snapshot.
+# Usage: is_service_external $service [$config_file]
+# Pass in the config file to check if the service is managed externally in the snapshot.
 is_service_external(){
   service=$1
   config_file=$2
@@ -401,11 +404,48 @@ is_service_external(){
         enabled=$(GIT_CONFIG="$config_file" git config mysql.external.enabled)
         [ "$enabled" == "true" ];
       else
-        ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.external.enabled"
+        ghe-ssh "$GHE_HOSTNAME" -- ghe-config --true "mysql.external.enabled"
       fi
       ;;
     *)
       return 1
       ;;
     esac
 }
+
+is_instance_configured(){
+  ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/configured' ]"
+}
+
+# Helper method that returns true if:
+# - the target appliance uses the internal MySQL database (aka NOT BYODB), and
+# - the snapshot being restored is from an appliance using an external MySQL database (BYODB)
+external_database_snapshot_to_internal_database(){
+  ! is_external_database_target && is_external_database_snapshot
+}
+
+# Helper method that returns true if:
+# - the target appliance uses an external MySQL database (BYODB), and
+# - the snapshot being restored is from an appliance using an internal MySQL database (aka NOT BYODB)
+internal_database_snapshot_to_external_database(){
+  is_external_database_target && ! is_external_database_snapshot
+}
+
+is_external_database_target_or_snapshot(){
+  # If restoring settings, only check if the snapshot being restored was from an appliance with external DB configured.
+  if $RESTORE_SETTINGS; then
+    is_external_database_snapshot
+  else
+    # Check if restoring a snapshot with an external database configured, or restoring
+    # to an appliance with an external database configured.
+    is_external_database_snapshot || is_external_database_target
+  fi
+}
+
+is_external_database_target(){
+  is_service_external "mysql"
+}
+
+is_external_database_snapshot(){
+  is_service_external "mysql" "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/settings.json"
+}
diff --git a/share/github-backup-utils/ghe-backup-mysql b/share/github-backup-utils/ghe-backup-mysql
@@ -15,7 +15,7 @@ bm_start "$(basename $0)"
 # Perform a host-check and establish the remote version in GHE_REMOTE_VERSION.
 ghe_remote_version_required "$GHE_HOSTNAME"
 
-if is_service_external 'mysql'; then
+if is_external_database_target; then
   echo "Backing up external MySQL database using customer-provided script..."
   if [ -n "$EXTERNAL_DATABASE_BACKUP_SCRIPT" ]; then
     $EXTERNAL_DATABASE_BACKUP_SCRIPT
diff --git a/share/github-backup-utils/ghe-restore-es-audit-log b/share/github-backup-utils/ghe-restore-es-audit-log
@@ -36,7 +36,7 @@ last_month=$(printf "audit_log(-[0-9]+)?-%4d-%02d(-[0-9]+)?" $last_yr $last_mth)
 current_month=$(printf "audit_log(-[0-9]+)?-%4d-%02d(-[0-9]+)?" $this_yr $this_mth)
 
 tmp_list="$(mktemp -t backup-utils-restore-XXXXXX)"
-if ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/configured' ]"; then
+if is_instance_configured; then
   configured=true
 fi
 
diff --git a/share/github-backup-utils/ghe-restore-es-hookshot b/share/github-backup-utils/ghe-restore-es-hookshot
@@ -24,7 +24,7 @@ last_index=$(ghe-ssh "$GHE_HOSTNAME" 'curl -s "localhost:9201/_cat/indices/hooks
 
 indices=$(find $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/hookshot/*.gz -print0 2>/dev/null | xargs -0 -I{} -n1 basename {} .gz)
 tmp_list="$(mktemp -t backup-utils-restore-XXXXXX)"
-if ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/configured' ]"; then
+if is_instance_configured; then
   configured=true
 fi
 
diff --git a/share/github-backup-utils/ghe-restore-external-database-compatibility-check b/share/github-backup-utils/ghe-restore-external-database-compatibility-check
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Usage: ghe-restore-external-database-compatibility-check
+# GitHub Enterprise checks for external-database related restores.
+
+# Bring in the backup configuration
+# shellcheck source=share/github-backup-utils/ghe-backup-config
+. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
+
+set -e
+
+# Always allow restoring to unconfigured appliances.
+# Additional checks are required if the instance is configured.
+if is_instance_configured; then
+
+  if internal_database_snapshot_to_external_database; then
+
+    # Restoring settings in this scenario would change BYODB state, which is not supported via backup-utils.
+    if $RESTORE_SETTINGS; then
+      echo "Restoring the settings of a snapshot from an appliance using the bundled MySQL service to an appliance using an externally-managed MySQL service is not supported."
+      echo "Please reconfigure the appliance first, then run ghe-restore again."
+      exit 1
+    fi
+
+    # Restoring interal DB snapshot to BYODB appliance without passing in --skip-mysql is not supported.
+    if ! $SKIP_MYSQL; then
+      echo "Restoring a snapshot from an appliance using the bundled MySQL service to an appliance using an externally-managed MySQL service is not supported."
+      echo "Please migrate the MySQL data beforehand, then run ghe-restore again, passing in the --skip-mysql flag."
+      exit 1
+    fi
+  fi
+
+  if external_database_snapshot_to_internal_database; then
+
+    # Restoring settings in this scenario would change BYODB state, which is not supported via backup-utils.
+    if $RESTORE_SETTINGS; then
+      echo "Restoring the settings of a snapshot from an appliance using an externally-managed MySQL service to an appliance using the bundled MySQL service is not supported."
+      echo "Please reconfigure the appliance first, then run ghe-restore again."
+      exit 1
+    fi
+
+    # Restoring BYODB snapshot to internal DB appliance without passing in --skip-mysql is not supported.
+    if ! $SKIP_MYSQL; then
+      echo "Restoring a snapshot from an appliance using an externally-managed MySQL service to an appliance using the bundled MySQL service is not supported."
+      echo "Please migrate the MySQL data beforehand, then run ghe-restore again, passing in the --skip-mysql flag."
+      exit 1
+    fi
+  fi
+fi
diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql
@@ -29,8 +29,7 @@ export GHE_RESTORE_SNAPSHOT
 # The directory holding the snapshot to restore
 snapshot_dir="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
 
-# When customer uses external database, we rely on customer to implement the restore process
-if is_service_external 'mysql' "$snapshot_dir/settings.json"; then
+if is_external_database_snapshot; then
   if [ -n "$EXTERNAL_DATABASE_RESTORE_SCRIPT" ]; then
     $EXTERNAL_DATABASE_RESTORE_SCRIPT
   else
diff --git a/test/bin/ghe-cluster-each b/test/bin/ghe-cluster-each
@@ -3,10 +3,37 @@
 # Emulates the remote GitHub ghe-config command. Tests use this
 # to assert that the command was executed.
 set -e
-if [[ "$*" =~ "-u" ]]; then
-  # Mimic `ghe-cluster-each $role -u`
-  echo "fake-uuid
-fake-uuid1
-fake-uuid2
-"
+
+for _ in "$@"; do
+  case "$1" in
+    -u)
+      SHOW_UUID=true
+      shift
+      ;;
+    -r|--role)
+        # fake change last save timestamp every 1s
+        ROLE=$2
+        shift
+        shift
+        ;;
+  esac
+done
+
+if $SHOW_UUID; then
+  CONFIG="$GHE_REMOTE_DATA_USER_DIR/common/cluster.conf"
+
+  hosts=$(git config -f $CONFIG --get-regexp cluster.*.hostname | cut -d ' ' -f2)
+
+  if [ -z "$hosts" ]; then
+    # Mimic `ghe-cluster-each $role -u`
+    echo "fake-uuid
+  fake-uuid1
+  fake-uuid2
+  "
+  else
+    for hostname in $hosts; do
+      [ -n "$ROLE" ] && [ "$(git config -f $CONFIG cluster.$hostname.$ROLE-server)" != "true" ] && continue
+      echo $hostname
+    done
+  fi
 fi
diff --git a/test/bin/ghe-config b/test/bin/ghe-config
@@ -9,4 +9,10 @@ if [ "$1" = "--true" ]; then
   [[ $($0 "$@") = true ]] && exit 0 || exit 1
 fi
 
-git config -f "$GHE_REMOTE_DATA_USER_DIR/common/secrets.conf" "$@"
+if [[ "$1" =~ secrets\..+ ]]; then
+  CONFIG="$GHE_REMOTE_DATA_USER_DIR/common/secrets.conf"
+else
+  CONFIG="$GHE_REMOTE_DATA_USER_DIR/common/github.conf"
+fi
+
+git config -f $CONFIG "$@"
diff --git a/test/test-ghe-restore-external-database.sh b/test/test-ghe-restore-external-database.sh
diff --git a/test/testlib.sh b/test/testlib.sh
