Skip to content

Commit 8f5d2d5

Browse files
jemrobinsonjemrobinson
authored
Merge pull request #545 from alan-turing-institute/DSG-2020-04-Deployment
Changes from April 2020 DSG
2 parents 0ec1eae + 3ea8d4b commit 8f5d2d5

27 files changed

Lines changed: 440 additions & 401 deletions

‎deployment/administration/SHM_Shutdown_Deployment_VMs.ps1‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,20 @@ Import-Module Az
77
Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
88
Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
99

10-
# Get SHM config
11-
$config = Get-ShmFullConfig($shmId)
1210

13-
# Temporarily switch to SHM subscription
14-
$prevContext = Get-AzContext
15-
$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
11+
# Get config and original context before changing subscription
12+
# ------------------------------------------------------------
13+
$config = Get-ShmFullConfig $shmId
14+
$originalContext = Get-AzContext
15+
$_ = Set-AzContext -SubscriptionId $config.subscriptionName
1616

17+
18+
# Stopping the deployment servers
19+
# -------------------------------
1720
Add-LogMessage -Level Info "Stopping all Deployment Servers"
1821
Get-AzVM -ResourceGroupName "RG_SHM_DEPLOYMENT_POOL" | Stop-AzVM -Force -NoWait
1922

23+
2024
# Switch back to original subscription
21-
$_ = Set-AzContext -Context $prevContext;
25+
# ------------------------------------
26+
$_ = Set-AzContext -Context $originalContext

‎deployment/administration/SHM_Shutdown_IAAM_VMs.ps1‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,23 @@ Import-Module Az
77
Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
88
Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
99

10-
# Get SHM config
11-
$config = Get-ShmFullConfig($shmId)
1210

13-
# Temporarily switch to SHM subscription
14-
$prevContext = Get-AzContext
15-
$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
11+
# Get config and original context before changing subscription
12+
# ------------------------------------------------------------
13+
$config = Get-ShmFullConfig $shmId
14+
$originalContext = Get-AzContext
15+
$_ = Set-AzContext -SubscriptionId $config.subscriptionName
1616

17+
18+
# Stop all IAAM VMs
19+
# -----------------
1720
Add-LogMessage -Level Info "Stopping NPS Server"
1821
Stop-AzVM -ResourceGroupName $config.nps.rg -Name $config.nps.vmName -Force -NoWait
1922
Add-LogMessage -Level Info "Stopping AD DCs"
2023
Stop-AzVM -ResourceGroupName $config.dc.rg -Name $config.dc.vmName -Force -NoWait
2124
Stop-AzVM -ResourceGroupName $config.dc.rg -Name $config.dcb.vmName -Force -NoWait
2225

26+
2327
# Switch back to original subscription
24-
$_ = Set-AzContext -Context $prevContext;
28+
# ------------------------------------
29+
$_ = Set-AzContext -Context $originalContext

‎deployment/administration/SHM_Shutdown_Mirror_VMs.ps1‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,19 @@ Import-Module Az
77
Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
88
Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
99

10-
# Get SHM config
11-
$config = Get-ShmFullConfig($shmId)
10+
# Get config and original context before changing subscription
11+
# ------------------------------------------------------------
12+
$config = Get-ShmFullConfig $shmId
13+
$originalContext = Get-AzContext
14+
$_ = Set-AzContext -SubscriptionId $config.subscriptionName
1215

13-
# Temporarily switch to SHM subscription
14-
$prevContext = Get-AzContext
15-
$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
1616

17-
Add-LogMessage -Level Info "Stopping all Mirror Servers"
17+
# Stopping the package mirrors
18+
# ----------------------------
19+
Add-LogMessage -Level Info "Stopping all package mirror servers"
1820
Get-AzVM -ResourceGroupName "RG_SHM_PKG_MIRRORS" | Stop-AzVM -Force -NoWait
1921

22+
2023
# Switch back to original subscription
21-
$_ = Set-AzContext -Context $prevContext;
24+
# ------------------------------------
25+
$_ = Set-AzContext -Context $originalContext

‎deployment/administration/SHM_Start_Deployment_VMs.ps1‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,23 @@ param(
55

66
Import-Module Az
77
Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
8+
Import-Module $PSScriptRoot/../common/Deployments.psm1 -Force
89
Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
910

10-
# Get SHM config
11-
$config = Get-ShmFullConfig($shmId)
1211

13-
# Temporarily switch to SHM subscription
14-
$prevContext = Get-AzContext
15-
$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
12+
# Get config and original context before changing subscription
13+
# ------------------------------------------------------------
14+
$config = Get-ShmFullConfig $shmId
15+
$originalContext = Get-AzContext
16+
$_ = Set-AzContext -SubscriptionId $config.subscriptionName
17+
18+
19+
# Start/restart the deployment servers
20+
# ------------------------------------
21+
Add-LogMessage -Level Info "Starting all Deployment Servers"
22+
Get-AzVM -ResourceGroupName "RG_SHM_DEPLOYMENT_POOL" | ForEach-Object { Enable-AzVM -Name $_.Name -ResourceGroupName $_.ResourceGroupName }
1623

17-
Add-LogMessage -Level Info "Stopping all Deployment Servers"
18-
Get-AzVM -ResourceGroupName "RG_SHM_DEPLOYMENT_POOL" | Restart-AzVM -NoWait
1924

2025
# Switch back to original subscription
21-
$_ = Set-AzContext -Context $prevContext;
26+
# ------------------------------------
27+
$_ = Set-AzContext -Context $originalContext

‎deployment/administration/SHM_Start_IAAM_VMs.ps1‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,28 @@ param(
55

66
Import-Module Az
77
Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
8+
Import-Module $PSScriptRoot/../common/Deployments.psm1 -Force
89
Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
910

10-
# Get SHM config
11-
$config = Get-ShmFullConfig($shmId)
1211

13-
# Temporarily switch to SHM subscription
14-
$prevContext = Get-AzContext
15-
$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
12+
# Get config and original context before changing subscription
13+
# ------------------------------------------------------------
14+
$config = Get-ShmFullConfig $shmId
15+
$originalContext = Get-AzContext
16+
$_ = Set-AzContext -SubscriptionId $config.subscriptionName
1617

18+
19+
# Start/restart all IAAM VMs
20+
# --------------------------
1721
Add-LogMessage -Level Info "Starting AD DCs..."
18-
Add-LogMessage -Level Info " - Waiting for Primary AD to start before starting other VMs."
19-
Restart-AzVM -ResourceGroupName $config.dc.rg -Name $config.dc.vmName
20-
Add-LogMessage -Level Info " - Waiting for Backup AD to start before starting other VMs."
21-
Restart-AzVM -ResourceGroupName $config.dc.rg -Name $config.dcb.vmName
22-
Add-LogMessage -Level Info "Starting NPS Server"
23-
Restart-AzVM -ResourceGroupName $config.nps.rg -Name $config.nps.vmName
22+
Add-LogMessage -Level Info "Starting Primary AD before other VMs..."
23+
Enable-AzVM -Name $config.dc.vmName -ResourceGroupName $config.dc.rg
24+
Add-LogMessage -Level Info "Starting Backup AD before other VMs..."
25+
Enable-AzVM -Name $config.dcb.vmName -ResourceGroupName $config.dc.rg
26+
Add-LogMessage -Level Info "Starting NPS Server..."
27+
Enable-AzVM -Name $config.nps.vmName -ResourceGroupName $config.nps.rg
28+
2429

2530
# Switch back to original subscription
26-
$_ = Set-AzContext -Context $prevContext;
31+
# ------------------------------------
32+
$_ = Set-AzContext -Context $originalContext

‎deployment/administration/SHM_Start_Mirror_VMs.ps1‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,23 @@ param(
55

66
Import-Module Az
77
Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
8+
Import-Module $PSScriptRoot/../common/Deployments.psm1 -Force
89
Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
910

10-
# Get SHM config
11-
$config = Get-ShmFullConfig($shmId)
1211

13-
# Temporarily switch to SHM subscription
14-
$prevContext = Get-AzContext
15-
$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
12+
# Get config and original context before changing subscription
13+
# ------------------------------------------------------------
14+
$config = Get-ShmFullConfig $shmId
15+
$originalContext = Get-AzContext
16+
$_ = Set-AzContext -SubscriptionId $config.subscriptionName
1617

18+
19+
# Start/restart the package mirrors
20+
# ---------------------------------
1721
Add-LogMessage -Level Info "Starting all Mirror Servers"
18-
Get-AzVM -ResourceGroupName "RG_SHM_PKG_MIRRORS" | Restart-AzVM -NoWait
22+
Get-AzVM -ResourceGroupName "RG_SHM_PKG_MIRRORS" | ForEach-Object { Enable-AzVM -Name $_.Name -ResourceGroupName $_.ResourceGroupName }
23+
1924

2025
# Switch back to original subscription
21-
$_ = Set-AzContext -Context $prevContext;
26+
# ------------------------------------
27+
$_ = Set-AzContext -Context $originalContext

‎deployment/administration/SRE_DSVM_Remote_Diagnostics.ps1‎

Lines changed: 85 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -9,48 +9,99 @@ Import-Module Az
99
Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
1010
Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
1111

12-
# Get SRE config
13-
# --------------
14-
$config = Get-SreConfig ($sreId);
12+
13+
# Get config and original context before changing subscription
14+
# ------------------------------------------------------------
15+
$config = Get-SreConfig $sreId
1516
$originalContext = Get-AzContext
16-
$_ = Set-AzContext -SubscriptionId $config.sre.subscriptionName;
17-
18-
19-
# Find VM with private IP address matching the provided last octect
20-
## Turn provided last octect into full IP address in the data subnet
21-
$vmIpAddress = ($config.sre.network.subnets.data.prefix + "." + $ipLastOctet)
22-
Add-LogMessage -Level Info "Finding VM with IP ${vmIpAddress}..."
23-
## Get all compute VMs
24-
$computeVms = Get-AzVM -ResourceGroupName $config.sre.dsvm.rg
25-
## Get the NICs attached to all the compute VMs
26-
$computeVmNicIds = ($computeVms | ForEach-Object { (Get-AzVM -ResourceGroupName $config.sre.dsvm.rg -Name $_.Name).NetworkProfile.NetworkInterfaces.Id })
27-
$computeVmNics = ($computeVmNicIds | ForEach-Object { Get-AzNetworkInterface -ResourceGroupName $config.sre.dsvm.rg -Name $_.Split("/")[-1] })
28-
## Filter the NICs to the one matching the desired IP address and get the name of the VM it is attached to
29-
$computeVmName = ($computeVmNics | Where-Object { $_.IpConfigurations.PrivateIpAddress -match $vmIpAddress })[0].VirtualMachine.Id.Split("/")[-1]
30-
31-
# Run remote scripts
32-
$diagnostic_scripts = @("check_ldap_connection.sh", "restart_name_resolution_service.sh", "rerun_realm_join.sh", "restart_sssd_service.sh")
33-
$testHost = $config.shm.dc.fqdn
34-
$ldapUser = $config.sre.users.ldap.dsvm.samAccountName
35-
$domainLower = $config.shm.domain.fqdn
36-
$servicePath = $config.shm.domain.serviceOuPath
17+
$_ = Set-AzContext -SubscriptionId $config.sre.subscriptionName
18+
19+
20+
# Find VM with private IP address matching the provided last octet
21+
# ----------------------------------------------------------------
22+
Add-LogMessage -Level Info "Finding compute VM with last IP octet: $ipLastOctet"
23+
$vmId = Get-AzNetworkInterface -ResourceGroupName $config.sre.dsvm.rg | Where-Object { ($_.IpConfigurations.PrivateIpAddress).Split(".") -eq $ipLastOctet } | ForEach-Object { $_.VirtualMachine.Id }
24+
$vm = Get-AzVM -ResourceGroupName $config.sre.dsvm.rg | Where-Object { $_.Id -eq $vmId }
25+
if ($?) {
26+
Add-LogMessage -Level Success "Found compute VM '$($vm.Name)'"
27+
} else {
28+
Add-LogMessage -Level Fatal "Could not find VM with last IP octet '$ipLastOctet'"
29+
}
3730

31+
# Run remote diagnostic scripts
32+
# -----------------------------
33+
Add-LogMessage -Level Info "Running diagnostic scripts on VM $($vm.Name)..."
3834
$params = @{
3935
TEST_HOST = $config.shm.dc.fqdn
4036
LDAP_USER = $config.sre.users.ldap.dsvm.samAccountName
4137
DOMAIN_LOWER = $config.shm.domain.fqdn
42-
SERVICE_PATH = "'" + $config.shm.domain.serviceOuPath + "'"
38+
SERVICE_PATH = "'$($config.shm.domain.serviceOuPath)'"
39+
}
40+
foreach ($scriptNamePair in (("LDAP connection", "check_ldap_connection.sh"),
41+
("name resolution", "restart_name_resolution_service.sh"),
42+
("realm join", "rerun_realm_join.sh"),
43+
("SSSD service", "restart_sssd_service.sh"))) {
44+
$name, $diagnostic_script = $scriptNamePair
45+
$scriptPath = Join-Path $PSScriptRoot ".." "secure_research_environment" "remote" "compute_vm" "scripts" $diagnostic_script
46+
Add-LogMessage -Level Info "[ ] Configuring $name ($diagnostic_script) on compute VM '$($vm.Name)'"
47+
$result = Invoke-RemoteScript -Shell "UnixShell" -ScriptPath $scriptPath -VMName $vm.Name -ResourceGroupName $config.sre.dsvm.rg -Parameter $params
48+
$success = $?
49+
Write-Output $result.Value
50+
if ($success) {
51+
Add-LogMessage -Level Success "Configuring $name on $($vm.Name) was successful"
52+
} else {
53+
Add-LogMessage -Level Failure "Configuring $name on $($vm.Name) failed!"
54+
}
55+
}
56+
57+
58+
# Get LDAP secret from the KeyVault
59+
# ---------------------------------
60+
Add-LogMessage -Level Info "[ ] Loading LDAP secret from key vault '$($config.sre.keyVault.name)'"
61+
$kvLdapPassword = (Get-AzKeyVaultSecret -VaultName $config.sre.keyVault.Name -Name $config.sre.keyVault.secretNames.dsvmLdapPassword).SecretValueText;
62+
if ($kvLdapPassword) {
63+
Add-LogMessage -Level Success "Found LDAP secret in the key vault"
64+
} else {
65+
Add-LogMessage -Level Fatal "Could not load LDAP secret from key vault '$($config.sre.keyVault.name)'"
4366
}
4467

45-
Add-LogMessage -Level Info "Running diagnostic scripts on VM ${computeVmName}..."
4668

47-
foreach ($diagnostic_script in $diagnostic_scripts) {
48-
$scriptPath = Join-Path $PSScriptRoot "remote_scripts" $diagnostic_script
49-
$result = Invoke-AzVMRunCommand -ResourceGroupName $config.sre.dsvm.rg -Name "$computeVmName" `
50-
-CommandId 'RunShellScript' -ScriptPath $scriptPath `
51-
-Parameter $params
52-
Write-Output $result.Value;
69+
# Set LDAP secret on the compute VM
70+
# ---------------------------------
71+
Add-LogMessage -Level Info "[ ] Setting LDAP secret on compute VM '$($vm.Name)'"
72+
$scriptPath = Join-Path $PSScriptRoot ".." "secure_research_environment" "remote" "compute_vm" "scripts" "reset_ldap_password.sh"
73+
$params = @{
74+
ldapPassword = "`"$kvLdapPassword`""
75+
}
76+
$result = Invoke-RemoteScript -Shell "UnixShell" -ScriptPath $scriptPath -VMName $vm.Name -ResourceGroupName $config.sre.dsvm.rg -Parameter $params
77+
$success = $?
78+
Write-Output $result.Value
79+
if ($success) {
80+
Add-LogMessage -Level Success "Setting LDAP secret on compute VM $($vm.Name) was successful"
81+
} else {
82+
Add-LogMessage -Level Fatal "Setting LDAP secret on compute VM $($vm.Name) failed!"
83+
}
84+
85+
86+
# Set LDAP secret in local Active Directory on the SHM DC
87+
# -------------------------------------------------------
88+
$_ = Set-AzContext -SubscriptionId $config.shm.subscriptionName
89+
$scriptPath = Join-Path $PSScriptRoot ".." "secure_research_environment" "remote" "compute_vm" "scripts" "ResetLdapPasswordOnAD.ps1"
90+
$params = @{
91+
samAccountName = "`"$($config.sre.users.ldap.dsvm.samAccountName)`""
92+
ldapPassword = "`"$kvLdapPassword`""
5393
}
94+
Add-LogMessage -Level Info "[ ] Setting LDAP secret in local AD on '$($config.shm.dc.vmName)'"
95+
$result = Invoke-RemoteScript -Shell "PowerShell" -ScriptPath $scriptPath -VMName $config.shm.dc.vmName -ResourceGroupName $config.shm.dc.rg -Parameter $params
96+
$success = $?
97+
Write-Output $result.Value
98+
if ($success) {
99+
Add-LogMessage -Level Success "Setting LDAP secret on SHM DC was successful"
100+
} else {
101+
Add-LogMessage -Level Fatal "Setting LDAP secret on SHM DC failed!"
102+
}
103+
54104

55-
# Switch back to previous subscription
56-
$_ = Set-AzContext -Context $originalContext;
105+
# Switch back to original subscription
106+
# ------------------------------------
107+
$_ = Set-AzContext -Context $originalContext

‎deployment/administration/SRE_DSVM_Reset_Ldap.ps1‎

Lines changed: 0 additions & 75 deletions
This file was deleted.

0 commit comments

Comments
 (0)