Merge pull request #444 from alan-turing-institute/DSGN-Bristol-AUG2019

Take changes from Bristol DSGN SHM/DSG deployment

Author: martintoreilly

.gitignore

@@ -5,4 +5,5 @@ new_dsg_environment/azure-vms/tests/logistic.csv
 
 # All secrets folders, no matter how nested
 secrets/
-temp/
+temp/
+.vscode

new_dsg_environment/azure-runbooks/dsg_build_instructions.md

@@ -333,7 +333,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Login with domain user `<shm-domain>\atiadmin` and the SHM DC admin password from the `sh-management-dc-admin-password` secret in the Safe Haven Management KeyVault
 
-- From the "Server Management" application, select `Tools -> Group Policy Management` then `Active Directory Domains and Trust`
+- From the "Server Management" application, select `Tools -> Active Directory Domains and Trust`
 
 - Right click the management domain name and select `Properties`
 
@@ -575,7 +575,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Install the packages present in the folder
 
-- **NOTE:** Intalling TexLive (`install-tl-windows-xxx`) will take about an hour to install (including downloading lots of files from the internet), so it is recommended to leave this until last and then continue with the remaining sections of this runbook while the TexLive installation completes.
+- **NOTE:** Installing TexLive (`install-tl-windows-xxx`) will take about an hour to install (including downloading lots of files from the internet), so it is recommended to leave this until last and then continue with the remaining sections of this runbook while the TexLive installation completes.
 
 - Once installed logout of the server
 

new_dsg_environment/azure-vms/README-parallel-deploy-using-azure-vms.md

@@ -1,9 +1,9 @@
 ## Make a new deployment VM
 ### Deploy a new VM
 - Go to the `new_dsg_environment/azure-vms` directory
-- Run `./deploy_azure_deployment_pool_vm.sh -s "Safe Haven Management Testing" -n <number>` where number is a zero padded number one greater than largest one currently used by an existing deployment VM (eg. 01, 02, etc.)
+- Run `./deploy_azure_deployment_pool_vm.sh -s "<sh-management-subscription-name>" -i <shm-id> -n <number>` where `<number>` is a zero padded number one greater than largest one currently used by an existing deployment VM (eg. 01, 02, etc.)
 - This will deploy a VM that you can use for deployment (ie. it has the necessary tools already installed)
-- You will need to take the public keypair output at the end of deployment and add it to the `Secrets > Deploy Keys` tab in the safe haven repo on GitHub.
+- You will need to take the public keypair output at the end of deployment and add it to the `Settings > Deploy Keys` tab in the safe haven repo on GitHub. The keys **do not** need write access, so leave this box **unchecked**. 
 
 ### VM information
 - The username is `atiadmin`
@@ -13,13 +13,14 @@
     - azure-cli
     - powershell
     - pip
+    - eternal terminal
 - You will need to clone the safe haven repo via `git clone git@github.com:alan-turing-institute/data-safe-haven.git`. This will automatically authenticate using the VM's `id_rsa` SSH key you just added to the safe haven repo as a deploy key.
 
 
 ## Use a deployment VM to deploy to the Safe Haven
 The VM(s) you want to use may be stopped to save money, so you may need to start the VM(s) you want to use from the Azure Portal
-- Install [mosh](https://mosh.org/) locally for more stable SSH (e.g. via `brew install mosh` on OSX)
-- Connect to the VM using `mosh atiadmin@sh-deployment-0X.westeurope.cloudapp.azure.com` (replacing `0X` with the zero padded number of the deployment VM you want to use and using the password from the `deployment-vm-admin-password` secret in `dsg-management-test` KeyVault in the `RG_DSG_SECRETS` resource group of the `Safe Haven Management Testing` subscription)
+- Connect to the VM using `ssh atiadmin@sh-deployment-<shm-id>-<number>.westeurope.cloudapp.azure.com` (replacing `0X` with the zero padded number of the deployment VM you want to use; and using the password from the `deployment-vm-admin-password` secret in `dsg-management-<shm-id>` KeyVault in the `RG_DSG_SECRETS` resource group of the `<sh-management-subscription-name>` subscription)
+- If you have not yet cloned the safe haven GitHub repository, run `git clone git@github.com:alan-turing-institute/data-safe-haven.git`
 - Navigate to the folder in the safe haven repo with the deployment scripts using `cd data-safe-haven/new_dsg_environment/dsg_deploy_scripts/07_deploy_compute_vms`
 - Checkout the master branch using `git checkout master` (or the deployment branch for the DSG environment you are deploying to - you may need to run `git fetch` first if not using `master`)
 - Ensure you have the latest changes locally using `git pull`

new_dsg_environment/azure-vms/cloud-init-deployment-vm.yaml

@@ -28,9 +28,17 @@ runcmd:
   - mkdir -p /home/atiadmin/.ssh
   - ssh-keygen -t rsa -b 4096 -N "" -f /home/atiadmin/.ssh/id_rsa
   - chown atiadmin:atiadmin /home/atiadmin/.ssh/id_rsa*
-  # Install Azure CLI support for Powershell
-  - pwsh -command "Install-Module -Name Az -Force -AllowClobber"
-  - pwsh -command "Install-Module -Name Az.Accounts -Force -AllowClobber"
+  # Install Azure support for Powershell (needs to install for the shared admin user, not root)
+  - sudo -u atiadmin pwsh -command "Install-Module -Name Az -Force -AllowClobber"
+
+  # Install Eternal Terminal
+  - apt-get install -y software-properties-common
+  - add-apt-repository ppa:jgmath2000/et
+  - apt-get update
+  - apt-get install et
+
+  # Install certbot
+  - apt install certbot
 
   # Clean up
   - apt-get -y upgrade

new_dsg_environment/azure-vms/deploy_azure_compute_vm.sh

@@ -407,8 +407,8 @@ LDAP_FILTER_ESCAPED=${LDAP_FILTER/"&"/"\&"}
 LDAP_FILTER_REGEX="s/LDAP_FILTER/${LDAP_FILTER_ESCAPED}/g"
 AD_DC_NAME_UPPER_REGEX="s/AD_DC_NAME_UPPER/${AD_DC_NAME_UPPER}/g"
 AD_DC_NAME_LOWER_REGEX="s/AD_DC_NAME_LOWER/${AD_DC_NAME_LOWER}/g"
-PYPI_MIRROR_IP_REGEX="s/PYPI_MIRROR_IP/${PYPI_MIRROR_IP}/"
-CRAN_MIRROR_IP_REGEX="s/CRAN_MIRROR_IP/${CRAN_MIRROR_IP}/"
+PYPI_MIRROR_IP_REGEX="s|PYPI_MIRROR_IP|${PYPI_MIRROR_IP}|g"
+CRAN_MIRROR_IP_REGEX="s|CRAN_MIRROR_IP|${CRAN_MIRROR_IP}|g"
 
 # Substitute regexes
 sed -e "${USERNAME_REGEX}" -e "${LDAP_SECRET_REGEX}" -e "${MACHINE_NAME_REGEX}" -e "${LDAP_USER_REGEX}" -e "${DOMAIN_LOWER_REGEX}" -e "${DOMAIN_UPPER_REGEX}" -e "${LDAP_CN_REGEX}" -e "${LDAP_BASE_DN_REGEX}" -e "${LDAP_FILTER_REGEX}" -e "${LDAP_BIND_DN_REGEX}" -e  "${AD_DC_NAME_UPPER_REGEX}" -e "${AD_DC_NAME_LOWER_REGEX}" -e "${PYPI_MIRROR_IP_REGEX}" -e "${CRAN_MIRROR_IP_REGEX}" $CLOUD_INIT_YAML > $TMP_CLOUD_CONFIG_YAML

new_dsg_environment/azure-vms/deploy_azure_deployment_pool_vm.sh

@@ -52,17 +52,21 @@ print_usage_and_exit() {
     echo "usage: $0 [-h] -s subscription -n machine_number [-r resource_group]"
     echo "  -h                                    display help"
     echo "  -s subscription [required]            specify subscription to deploy into. (Test using 'Safe Haven Management Testing')"
+    echo "  -i shm_id [required]                  specify the short ID for the Safe Haven Management segment (e.g. prod, test etc)"
     echo "  -n machine_number [required]          specify number of created VM, which must be unique in this resource group (VM will be called '${MACHINENAMEPREFIX}-<number>')"
     echo "  -r resource_group                     specify resource group for deploying the VM image - will be created if it does not already exist (defaults to '${RESOURCEGROUP}')"
     exit 1
 }
 
 # Read command line arguments, overriding defaults where necessary
-while getopts "hn:r:s:" opt; do
+while getopts "hi:n:r:s:" opt; do
     case $opt in
         h)
             print_usage_and_exit
             ;;
+        i)
+            SHMID=$OPTARG
+            ;;
         n)
             MACHINENUMBER=$OPTARG
             ;;
@@ -87,13 +91,19 @@ if [ "$SUBSCRIPTION" = "" ]; then
 fi
 
 
-# Check that a machine name has been provided
+# Check that an SHM ID has been provided
+# -------------------------------------------
+if [ "$SHMID" = "" ]; then
+    echo -e "${RED}SHM ID is a required argument!${END}"
+    print_usage_and_exit
+fi
+# Check that a machine number has been provided
 # -------------------------------------------
 if [ "$MACHINENUMBER" = "" ]; then
     echo -e "${RED}Machine number is a required argument!${END}"
     print_usage_and_exit
 fi
-MACHINENAME="${MACHINENAMEPREFIX}-${MACHINENUMBER}"
+MACHINENAME="${MACHINENAMEPREFIX}-${SHMID}-${MACHINENUMBER}"
 DNSNAME="$(echo $MACHINENAME | tr '[:upper:]' '[:lower:]')"
 
 
@@ -129,8 +139,10 @@ fi
 if [ "$(az network nsg show --resource-group $RESOURCEGROUP --name $NSG_NAME 2> /dev/null)" = "" ]; then
     echo -e "${BOLD}Creating NSG for deployment: ${BLUE}$NSG_NAME${END}"
     az network nsg create --resource-group $RESOURCEGROUP --name $NSG_NAME
-    az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_NAME --direction Inbound --priority 1000 --name AllowMosh --description "Allow Mosh" --access "Allow" --source-address-prefixes "*" --source-port-ranges "*" --destination-address-prefixes "VirtualNetwork" --destination-port-ranges "60000-61000" --protocol "UDP"
-    az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_NAME --direction Inbound --priority 2000 --name AllowTuringIPs --description "Allow inbound Turing connections on port 22 (SSH)" --access "Allow" --source-address-prefixes 193.60.220.253 193.60.220.240 --source-port-ranges "*" --destination-address-prefixes "*" --destination-port-ranges "22" --protocol "TCP"
+    PERMITTED_IPS="193.60.220.253 193.60.220.240"
+    az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_NAME --direction Inbound --priority 1000 --name AllowMosh --description "Allow Mosh" --access "Allow" --source-address-prefixes "$PERMITTED_IPS" --source-port-ranges "*" --destination-address-prefixes "*" --destination-port-ranges "60000-61000" --protocol "UDP"
+    az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_NAME --direction Inbound --priority 1500 --name AllowEternalTerminal --description "Allow Eternal Terminal" --access "Allow" --source-address-prefixes "$PERMITTED_IPS" --source-port-ranges "*" --destination-address-prefixes "*" --destination-port-ranges "2022" --protocol "TCP"
+    az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_NAME --direction Inbound --priority 2000 --name AllowTuringIPs --description "Allow inbound Turing connections on port 22 (SSH)" --access "Allow" --source-address-prefixes "$PERMITTED_IPS" --source-port-ranges "*" --destination-address-prefixes "*" --destination-port-ranges "22" --protocol "TCP"
     az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_NAME --direction Inbound --priority 3000 --name IgnoreInboundRulesBelowHere --description "Deny all other inbound" --access "Deny" --source-address-prefixes "*" --source-port-ranges "*" --destination-address-prefixes "*" --destination-port-ranges "*" --protocol "*"
 fi
 
@@ -189,7 +201,7 @@ else
         --os-disk-name $OSDISKNAME \
         --public-ip-address-dns-name $DNSNAME \
         --resource-group $RESOURCEGROUP \
-        --size Standard_F4s_v2 \
+        --size Standard_B2ms \
         --storage-sku Standard_LRS \
         --subnet $SUBNET_NAME \
         --vnet-name $VNET_NAME

new_dsg_environment/azure-vms/deploy_azure_external_mirror_servers.sh

@@ -185,7 +185,7 @@ else
 
     # Temporarily allow outbound internet connections through the NSG from this IP address only
     PRIVATEIPADDRESS=${VNET_IPTRIPLET}.4
-    echo -e "${BOLD}Temporarily allowing outbound internet access from ${BLUE}$PRIVATEIPADDRESS${END}${BOLD} in NSG ${BLUE}$NSG_EXTERNAL${END}${BOLD} (for use during deployment *only*)${END}"
+    echo -e "${BOLD}Temporarily allowing outbound internet access on ports 80, 443 and 3128 from ${BLUE}$PRIVATEIPADDRESS${END}${BOLD} in NSG ${BLUE}$NSG_EXTERNAL${END}${BOLD} (for installing software during deployment *only*)${END}"
     az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_EXTERNAL --direction Outbound --name configurationOutboundTemporary --description "Allow ports 80 (http), 443 (pip) and 3128 (pip) for installing software" --access "Allow" --source-address-prefixes $PRIVATEIPADDRESS --destination-port-ranges 80 443 3128 --protocol TCP --destination-address-prefixes Internet --priority 100 --output none
     az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_EXTERNAL --direction Outbound --name vnetOutboundTemporary --description "Block connections to the VNet" --access "Deny" --source-address-prefixes $PRIVATEIPADDRESS --destination-port-ranges "*" --protocol "*" --destination-address-prefixes VirtualNetwork --priority 150 --output none
 
@@ -215,14 +215,10 @@ else
 
     # Poll VM to see whether it has finished running
     echo -e "${BOLD}Waiting for VM setup to finish (this may take several minutes)...${END}"
-    while true; do
-        POLL=$(az vm get-instance-view --resource-group $RESOURCEGROUP --name $MACHINENAME --query "instanceView.statuses[?code == 'PowerState/running'].displayStatus")
-        if [ "$(echo $POLL | grep 'VM running')" == "" ]; then break; fi
-        sleep 10
-    done
+    az vm wait --name $MACHINENAME --resource-group $RESOURCEGROUP --custom "instanceView.statuses[?code == 'PowerState/stopped'].displayStatus" --output none
 
     # Delete the configuration NSG rule and restart the VM
-    echo -e "${BOLD}Restarting VM: ${BLUE}${MACHINENAME}${END}" --output none
+    echo -e "${BOLD}Restarting VM: ${BLUE}${MACHINENAME}${END}"
     az network nsg rule delete --resource-group $RESOURCEGROUP --nsg-name $NSG_EXTERNAL --name configurationOutboundTemporary --output none
     az network nsg rule delete --resource-group $RESOURCEGROUP --nsg-name $NSG_EXTERNAL --name vnetOutboundTemporary --output none
     az vm start --resource-group $RESOURCEGROUP --name $MACHINENAME --output none
@@ -271,7 +267,7 @@ if [ "$TIER" == "2" ]; then  # we do not support Tier-3 CRAN mirrors at present
 
         # Temporarily allow outbound internet connections through the NSG from this IP address only
         PRIVATEIPADDRESS=${VNET_IPTRIPLET}.5
-        echo -e "${BOLD}Temporarily allowing outbound internet access from ${BLUE}$PRIVATEIPADDRESS${END}${BOLD} in NSG ${BLUE}$NSG_EXTERNAL${END}${BOLD} (for use during deployment *only*)${END}"
+        echo -e "${BOLD}Temporarily allowing outbound internet access on ports 80, 443 and 3128 from ${BLUE}$PRIVATEIPADDRESS${END}${BOLD} in NSG ${BLUE}$NSG_EXTERNAL${END}${BOLD} (for installing software during deployment *only*)${END}"
         az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_EXTERNAL --direction Outbound --name configurationOutboundTemporary --description "Allow ports 80 (http), 443 (pip) and 3128 (pip) for installing software" --access "Allow" --source-address-prefixes $PRIVATEIPADDRESS --destination-port-ranges 80 443 3128 --protocol TCP --destination-address-prefixes Internet --priority 100 --output none
         az network nsg rule create --resource-group $RESOURCEGROUP --nsg-name $NSG_EXTERNAL --direction Outbound --name vnetOutboundTemporary --description "Block connections to the VNet" --access "Deny" --source-address-prefixes $PRIVATEIPADDRESS --destination-port-ranges "*" --protocol "*" --destination-address-prefixes VirtualNetwork --priority 200 --output none
 
@@ -300,11 +296,7 @@ if [ "$TIER" == "2" ]; then  # we do not support Tier-3 CRAN mirrors at present
 
         # Poll VM to see whether it has finished running
         echo -e "${BOLD}Waiting for VM setup to finish (this may take several minutes)...${END}"
-        while true; do
-            POLL=$(az vm get-instance-view --resource-group $RESOURCEGROUP --name $MACHINENAME --query "instanceView.statuses[?code == 'PowerState/running'].displayStatus")
-            if [ "$(echo $POLL | grep 'VM running')" == "" ]; then break; fi
-            sleep 10
-        done
+        az vm wait --name $MACHINENAME --resource-group $RESOURCEGROUP --custom "instanceView.statuses[?code == 'PowerState/stopped'].displayStatus" --output none
 
         # Delete the configuration NSG rule and restart the VM
         echo -e "${BOLD}Restarting VM: ${BLUE}${MACHINENAME}${END}"