Skip to content

Commit 0ec1eae

Browse files
jemrobinsonjemrobinson
authored
Merge pull request #533 from alan-turing-institute/530-validate-infrastructure
Programatically validate infrastructure
2 parents ff83ef1 + a5a51c8 commit 0ec1eae

3 files changed

Lines changed: 4433 additions & 0 deletions

File tree

‎tests/Compare_Deployments.ps1‎

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
# Parameter sets in Powershell are a bit counter-intuitive. See here (https://docs.microsoft.com/en-us/powershell/scripting/developer/cmdlet/cmdlet-parameter-sets?view=powershell-7) for details
2+
param(
3+
[Parameter(Mandatory = $true, HelpMessage = "Name of the test (proposed) subscription")]
4+
[string]$Subscription,
5+
[Parameter(ParameterSetName="BenchmarkSubscription", Mandatory = $true, HelpMessage = "Name of the benchmark subscription to compare against")]
6+
[string]$BenchmarkSubscription,
7+
[Parameter(ParameterSetName="BenchmarkConfig", Mandatory = $true, HelpMessage = "Path to the benchmark config to compare against")]
8+
[string]$BenchmarkConfig,
9+
[Parameter(Mandatory = $false, HelpMessage = "Print verbose logging messages")]
10+
[switch]$VerboseLogging = $false
11+
)
12+
13+
# Install required modules
14+
if (-Not $(Get-Module -ListAvailable -Name Az)) { Install-Package Az -Force}
15+
if (-Not $(Get-Module -ListAvailable -Name Communary.PASM)) { Install-Package Communary.PASM -Force}
16+
17+
# Import modules
18+
Import-Module Az
19+
Import-Module Communary.PASM
20+
Import-Module $PSScriptRoot/../common_powershell/Logging.psm1 -Force
21+
22+
function Select-ClosestMatch {
23+
param (
24+
[Parameter(Position = 0)][ValidateNotNullOrEmpty()]
25+
[string] $Value,
26+
[Parameter(Position = 1)][ValidateNotNullOrEmpty()]
27+
[System.Array] $Array
28+
)
29+
$Array | Sort-Object @{Expression={ Get-PasmScore -String1 $Value -String2 $_ -Algorithm "LevenshteinDistance" }; Ascending=$false} | Select-Object -First 1
30+
}
31+
32+
# Compare two NSG rule sets
33+
# Match parameter-by-parameter
34+
# --------------------------------------------
35+
function Compare-NSGRules {
36+
param (
37+
[Parameter()]
38+
[System.Array] $BenchmarkRules,
39+
[Parameter()]
40+
[System.Array] $TestRules
41+
)
42+
$nMatched = 0
43+
$unmatched = @()
44+
foreach ($benchmarkRule in $BenchmarkRules) {
45+
$lowestDifference = [double]::PositiveInfinity
46+
$closestMatchingRule = $null
47+
# Iterate over TestRules checking for an identical match by checking how many of the rule parameters differ
48+
# If an exact match is found then increment the counter, otherwise log the rule and the closest match
49+
foreach ($testRule in $TestRules) {
50+
$difference = 0
51+
if ($benchmarkRule.Protocol -ne $testRule.Protocol) { $difference += 1 }
52+
if ([string]($benchmarkRule.SourcePortRange) -ne [string]($testRule.SourcePortRange)) { $difference += 1 }
53+
if ([string]($benchmarkRule.DestinationPortRange) -ne [string]($testRule.DestinationPortRange)) { $difference += 1 }
54+
if ([string]($benchmarkRule.SourceAddressPrefix) -ne [string]($testRule.SourceAddressPrefix)) { $difference += 1 }
55+
if ([string]($benchmarkRule.DestinationAddressPrefix) -ne [string]($testRule.DestinationAddressPrefix)) { $difference += 1 }
56+
if ($benchmarkRule.Access -ne $testRule.Access) { $difference += 1 }
57+
if ($benchmarkRule.Priority -ne $testRule.Priority) { $difference += 1 }
58+
if ($benchmarkRule.Direction -ne $testRule.Direction) { $difference += 1 }
59+
if ($difference -lt $lowestDifference) {
60+
$lowestDifference = $difference
61+
$closestMatchingRule = $testRule
62+
}
63+
if ($difference -eq 0) { break }
64+
}
65+
66+
if ($lowestDifference -eq 0) {
67+
$nMatched += 1
68+
if ($VerboseLogging) { Add-LogMessage -Level Info "Found matching rule for $($benchmarkRule.Name)" }
69+
} else {
70+
Add-LogMessage -Level Error "Could not find an identical rule for $($benchmarkRule.Name)"
71+
$unmatched += $benchmarkRule.Name
72+
$benchmarkRule | Out-String
73+
Add-LogMessage -Level Info "Closest match was:"
74+
$closestMatchingRule | Out-String
75+
}
76+
}
77+
78+
$nTotal = $nMatched + $unmatched.Count
79+
if ($nMatched -eq $nTotal) {
80+
Add-LogMessage -Level Success "Matched $nMatched/$nTotal rules"
81+
} else {
82+
Add-LogMessage -Level Failure "Matched $nMatched/$nTotal rules"
83+
}
84+
}
85+
86+
87+
function Test-OutboundConnection {
88+
param (
89+
[Parameter(Position = 0)][ValidateNotNullOrEmpty()]
90+
[Microsoft.Azure.Commands.Compute.Models.PSVirtualMachine] $VM,
91+
[Parameter(Position = 0)][ValidateNotNullOrEmpty()]
92+
[string] $DestinationAddress,
93+
[Parameter(Position = 1)][ValidateNotNullOrEmpty()]
94+
[string] $DestinationPort
95+
)
96+
# Get the network watcher, creating a new one if required
97+
$networkWatcher = Get-AzNetworkWatcher | Where-Object { $_.Location -eq $VM.Location }
98+
if (-Not $networkWatcher) {
99+
$networkWatcher = New-AzNetworkWatcher -Name "NetworkWatcher" -ResourceGroupName "NetworkWatcherRG" -Location $VM.Location
100+
}
101+
# Ensure that the VM has the extension installed (if we have permissions for this)
102+
$networkWatcherExtension = Get-AzVMExtension -ResourceGroupName $VM.ResourceGroupName -VMName $VM.Name | Where-Object { ($_.Publisher -eq "Microsoft.Azure.NetworkWatcher") -and ($_.ProvisioningState -eq "Succeeded") }
103+
if (-Not $networkWatcherExtension) {
104+
Add-LogMessage -Level Info "... registering the Azure NetworkWatcher extension on $($VM.Name). "
105+
# Add the Windows extension
106+
if ($VM.OSProfile.WindowsConfiguration) {
107+
$_ = Set-AzVMExtension -ResourceGroupName $VM.ResourceGroupName -VMName $VM.Name -Location $VM.Location -Name "AzureNetworkWatcherExtension" -Publisher "Microsoft.Azure.NetworkWatcher" -Type "NetworkWatcherAgentWindows" -TypeHandlerVersion "1.4" -ErrorVariable NotInstalled -ErrorAction SilentlyContinue
108+
if ($NotInstalled) {
109+
Add-LogMessage -Level Warning "Unable to register Windows network watcher extension for $($VM.Name)"
110+
return "Unknown"
111+
}
112+
}
113+
# Add the Linux extension
114+
if ($VM.OSProfile.LinuxConfiguration) {
115+
$_ = Set-AzVMExtension -ResourceGroupName $VM.ResourceGroupName -VMName $VM.Name -Location $VM.Location -Name "AzureNetworkWatcherExtension" -Publisher "Microsoft.Azure.NetworkWatcher" -Type "NetworkWatcherAgentLinux" -TypeHandlerVersion "1.4" -ErrorVariable NotInstalled -ErrorAction SilentlyContinue
116+
if ($NotInstalled) {
117+
Add-LogMessage -Level Warning "Unable to register Linux network watcher extension for $($VM.Name)"
118+
return "Unknown"
119+
}
120+
}
121+
}
122+
Add-LogMessage -Level Info "... testing connectivity on port $DestinationPort"
123+
$networkCheck = Test-AzNetworkWatcherConnectivity -NetworkWatcher $networkWatcher -SourceId $VM.Id -DestinationAddress $DestinationAddress -DestinationPort $DestinationPort -ErrorVariable NotAvailable -ErrorAction SilentlyContinue
124+
if ($NotAvailable) {
125+
Add-LogMessage -Level Warning "Unable to test connection for $($VM.Name)"
126+
return "Unknown"
127+
} else {
128+
return $networkCheck.ConnectionStatus
129+
}
130+
}
131+
132+
function Convert-RuleToEffectiveRule {
133+
param (
134+
[Parameter(Position = 0)][ValidateNotNullOrEmpty()]
135+
[System.Object] $rule
136+
)
137+
$effectiveRule = [Microsoft.Azure.Commands.Network.Models.PSEffectiveSecurityRule]::new()
138+
$effectiveRule.Name = $rule.Name
139+
$effectiveRule.Protocol = $rule.Protocol.Replace("*", "All")
140+
# Source port range
141+
$effectiveRule.SourcePortRange = New-Object System.Collections.Generic.List[string]
142+
foreach ($port in $rule.SourcePortRange) {
143+
# We do not explicitly deal with the case where the port is not an integer, a range or '*'
144+
if ($port -eq "*") { $effectiveRule.SourcePortRange.Add("0-65535"); break }
145+
elseif ($port.Contains("-")) { $effectiveRule.SourcePortRange.Add($port) }
146+
else { $effectiveRule.SourcePortRange.Add("$port-$port") }
147+
}
148+
# Destination port range
149+
$effectiveRule.DestinationPortRange = New-Object System.Collections.Generic.List[string]
150+
foreach ($port in $rule.DestinationPortRange) {
151+
# We do not explicitly deal with the case where the port is not an integer, a range or '*'
152+
if ($port -eq "*") { $effectiveRule.DestinationPortRange.Add("0-65535"); break }
153+
elseif ($port.Contains("-")) { $effectiveRule.DestinationPortRange.Add($port) }
154+
else { $effectiveRule.DestinationPortRange.Add("$port-$port") }
155+
}
156+
# Source address prefix
157+
$effectiveRule.SourceAddressPrefix = New-Object System.Collections.Generic.List[string]
158+
foreach ($prefix in $rule.SourceAddressPrefix) {
159+
if ($prefix -eq "0.0.0.0/0") { $effectiveRule.SourceAddressPrefix.Add("*"); break }
160+
else { $effectiveRule.SourceAddressPrefix.Add($rule.SourceAddressPrefix) }
161+
}
162+
# Destination address prefix
163+
$effectiveRule.DestinationAddressPrefix = New-Object System.Collections.Generic.List[string]
164+
foreach ($prefix in $rule.DestinationAddressPrefix) {
165+
if ($prefix -eq "0.0.0.0/0") { $effectiveRule.DestinationAddressPrefix.Add("*"); break }
166+
else { $effectiveRule.DestinationAddressPrefix.Add($rule.DestinationAddressPrefix) }
167+
}
168+
$effectiveRule.Access = $rule.Access
169+
$effectiveRule.Priority = $rule.Priority
170+
$effectiveRule.Direction = $rule.Direction
171+
return $effectiveRule
172+
}
173+
174+
175+
function Get-NSGRules {
176+
param (
177+
[Parameter(Position = 0)][ValidateNotNullOrEmpty()]
178+
[Microsoft.Azure.Commands.Compute.Models.PSVirtualMachine] $VM
179+
)
180+
$effectiveNSG = Get-AzEffectiveNetworkSecurityGroup -NetworkInterfaceName ($VM.NetworkProfile.NetworkInterfaces.Id -Split '/')[-1] -ResourceGroupName $VM.ResourceGroupName -ErrorVariable NotAvailable -ErrorAction SilentlyContinue
181+
if ($NotAvailable) {
182+
# Not able to get effective rules so we'll construct them by hand
183+
$rules = @()
184+
# Get rules from NSG directly attached to the NIC
185+
$nic = Get-AzNetworkInterface | Where-Object { $_.Id -eq $VM.NetworkProfile.NetworkInterfaces.Id }
186+
$directNsgs = Get-AzNetworkSecurityGroup | Where-Object { $_.Id -eq $nic.NetworkSecurityGroup.Id }
187+
$directNsgRules = @()
188+
foreach ($directNsg in $directNsgs) {
189+
$directNsgRules = $directNsgRules + $directNsg.SecurityRules + $directNsg.DefaultSecurityRules
190+
}
191+
# Get rules from NSG attached to the subnet
192+
$subnetNsgs = Get-AzNetworkSecurityGroup | Where-Object { $_.Subnets.Id -eq $nic.IpConfigurations.Subnet.Id }
193+
$subnetNsgRules = @()
194+
foreach ($subnetNsg in $subnetNsgs) {
195+
$subnetNsgRules = $subnetNsgRules + $subnetNsg.SecurityRules + $subnetNsg.DefaultSecurityRules
196+
}
197+
$effectiveRules = @()
198+
if ($directNsgRules.Count -And $subnetNsgRules.Count) {
199+
Add-LogMessage -Level Warning "Found both NSG rules from both the NIC and the subnet for $($VM.Name). Evaluation of effective rules may be incorrect!"
200+
}
201+
# Convert each PSSecurityRule into a PSEffectiveSecurityRule
202+
foreach ($rule in ($directNsgRules + $subnetNsgRules)) {
203+
$effectiveRules = $effectiveRules + $(Convert-RuleToEffectiveRule $rule)
204+
}
205+
return $effectiveRules
206+
} else {
207+
$effectiveRules = $effectiveNSG.EffectiveSecurityRules
208+
# Sometimes the address prefix is retrieved as ("0.0.0.0/0", "0.0.0.0/0") rather than "*" (although these mean the same thing)
209+
foreach ($effectiveRule in $effectiveRules) {
210+
if ($effectiveRule.SourceAddressPrefix[0] -eq "0.0.0.0/0") { $effectiveRule.SourceAddressPrefix.Clear(); $effectiveRule.SourceAddressPrefix.Add("*") }
211+
if ($effectiveRule.DestinationAddressPrefix[0] -eq "0.0.0.0/0") { $effectiveRule.DestinationAddressPrefix.Clear(); $effectiveRule.DestinationAddressPrefix.Add("*") }
212+
}
213+
return $effectiveRules
214+
}
215+
}
216+
217+
# Get original context before switching subscription
218+
# --------------------------------------------------
219+
$originalContext = Get-AzContext
220+
221+
222+
# Load configuration from a benchmark subscription or config
223+
# ----------------------------------------------------------
224+
if ($BenchmarkSubscription) {
225+
$JsonConfig = [ordered]@{}
226+
# Get VMs in current subscription
227+
$_ = Set-AzContext -SubscriptionId $BenchmarkSubscription
228+
$benchmarkVMs = Get-AzVM | Where-Object { $_.Name -NotLike "*shm-deploy*" }
229+
Add-LogMessage -Level Info "Found $($benchmarkVMs.Count) VMs in subscription: '$BenchmarkSubscription'"
230+
foreach ($VM in $benchmarkVMs) {
231+
Add-LogMessage -Level Info "... $($VM.Name)"
232+
}
233+
# Get the NSG rules and connectivity for each VM in the subscription
234+
foreach ($benchmarkVM in $benchmarkVMs) {
235+
Add-LogMessage -Level Info "Getting NSG rules and connectivity for $($VM.Name)"
236+
$JsonConfig[$benchmarkVM.Name] = [ordered]@{
237+
InternetFromPort = [ordered]@{
238+
"80" = (Test-OutboundConnection -VM $benchmarkVM -DestinationAddress "google.com" -DestinationPort 80)
239+
"443" = (Test-OutboundConnection -VM $benchmarkVM -DestinationAddress "google.com" -DestinationPort 443)
240+
}
241+
Rules = Get-NSGRules -VM $benchmarkVM
242+
}
243+
}
244+
$OutputFile = New-TemporaryFile
245+
Out-File -FilePath $OutputFile -Encoding "UTF8" -InputObject ($JsonConfig | ConvertTo-Json -Depth 10)
246+
Add-LogMessage -Level Info "Configuration file generated at '$($OutputFile.FullName)'"
247+
$BenchmarkJsonPath = $OutputFile.FullName
248+
} elseif ($BenchmarkConfig) {
249+
$BenchmarkJsonPath = $BenchmarkConfig
250+
}
251+
252+
253+
# Deserialise VMs from JSON config
254+
# --------------------------------
255+
$BenchmarkJsonConfig = Get-Content -Path $BenchmarkJsonPath -Raw -Encoding UTF-8 | ConvertFrom-Json
256+
$benchmarkVMs = @()
257+
foreach ($JsonVm in $BenchmarkJsonConfig.PSObject.Properties) {
258+
$VM = New-Object -TypeName PsObject
259+
$VM | Add-Member -MemberType NoteProperty -Name Name -Value $JsonVm.Name
260+
$VM | Add-Member -MemberType NoteProperty -Name InternetFromPort -Value @{}
261+
$VM.InternetFromPort.80 = $JsonVm.PSObject.Properties.Value.InternetFromPort.80
262+
$VM.InternetFromPort.443 = $JsonVm.PSObject.Properties.Value.InternetFromPort.443
263+
$VM | Add-Member -MemberType NoteProperty -Name Rules -Value @()
264+
foreach ($rule in $JsonVm.PSObject.Properties.Value.Rules) {
265+
if ($rule.Name) { $VM.Rules += $(Convert-RuleToEffectiveRule $rule) }
266+
}
267+
$benchmarkVMs += $VM
268+
}
269+
270+
271+
# Get VMs in test SHM
272+
# -------------------
273+
$_ = Set-AzContext -SubscriptionId $Subscription
274+
$testVMs = Get-AzVM
275+
Add-LogMessage -Level Info "Found $($testVMs.Count) VMs in subscription: '$Subscription'"
276+
foreach ($VM in $testVMs) {
277+
Add-LogMessage -Level Info "... $($VM.Name)"
278+
}
279+
280+
281+
# Create a hash table which maps test VMs to benchmark ones
282+
# ---------------------------------------------------------
283+
$vmHashTable = @{}
284+
foreach ($testVM in $testVMs) {
285+
$nameToCheck = $testVM.Name
286+
# Override matches for names that would otherwise fail
287+
if ($nameToCheck.StartsWith("CRAN-EXTERNAL-MIRROR")) { $nameToCheck = $nameToCheck.Replace("CRAN-EXTERNAL-MIRROR", "CRAN-MIRROR-EXTERNAL") }
288+
if ($nameToCheck.StartsWith("CRAN-INTERNAL-MIRROR")) { $nameToCheck = $nameToCheck.Replace("CRAN-INTERNAL-MIRROR", "CRAN-MIRROR-INTERNAL") }
289+
if ($nameToCheck.StartsWith("PYPI-EXTERNAL-MIRROR")) { $nameToCheck = $nameToCheck.Replace("PYPI-EXTERNAL-MIRROR", "PYPI-MIRROR-EXTERNAL") }
290+
if ($nameToCheck.StartsWith("PYPI-INTERNAL-MIRROR")) { $nameToCheck = $nameToCheck.Replace("PYPI-INTERNAL-MIRROR", "PYPI-MIRROR-INTERNAL") }
291+
if ($nameToCheck.StartsWith("APP-SRE")) { $nameToCheck = $nameToCheck.Replace("APP-SRE", "RDSSH1") }
292+
if ($nameToCheck.StartsWith("DKP-SRE")) { $nameToCheck = $nameToCheck.Replace("DKP-SRE", "RDSSH2") }
293+
# Only match against names that have not been matched yet
294+
$benchmarkVMNames = $benchmarkVMs | ForEach-Object { $_.Name } | Where-Object { ($vmHashTable.Values | ForEach-Object { $_.Name }) -NotContains $_ }
295+
$benchmarkVM = $benchmarkVMs | Where-Object { $_.Name -eq $(Select-ClosestMatch -Array $benchmarkVMNames -Value $nameToCheck) }
296+
$vmHashTable[$testVM] = $benchmarkVM
297+
Add-LogMessage -Level Info "matched $($testVM.Name) => $($benchmarkVM.Name)"
298+
}
299+
300+
# Iterate over paired VMs checking their network settings
301+
# -------------------------------------------------------
302+
foreach ($testVM in $testVMs) {
303+
$benchmarkVM = $vmHashTable[$testVM]
304+
305+
# Get parameters for new VM
306+
# -------------------------
307+
$_ = Set-AzContext -SubscriptionId $Subscription
308+
Add-LogMessage -Level Info "Getting NSG rules and connectivity for $($testVM.Name)"
309+
$testRules = Get-NSGRules -VM $testVM
310+
# Check that each NSG rule has a matching equivalent (which might be named differently)
311+
Add-LogMessage -Level Info "Comparing NSG rules for $($benchmarkVM.Name) and $($testVM.Name)"
312+
Add-LogMessage -Level Info "... ensuring that all $($benchmarkVM.Name) rules exist on $($testVM.Name)"
313+
Compare-NSGRules -BenchmarkRules $benchmarkVM.Rules -TestRules $testRules
314+
Add-LogMessage -Level Info "... ensuring that all $($testVM.Name) rules exist on $($benchmarkVM.Name)"
315+
Compare-NSGRules -BenchmarkRules $testRules -TestRules $benchmarkVM.Rules
316+
317+
# Check that internet connectivity is the same for matched VMs
318+
Add-LogMessage -Level Info "Comparing internet connectivity for $($benchmarkVM.Name) and $($testVM.Name)..."
319+
# Test internet access on ports 80 and 443
320+
foreach ($port in (80, 443)) {
321+
$testInternet = Test-OutboundConnection -VM $testVM -DestinationAddress "google.com" -DestinationPort $port
322+
if ($benchmarkVM.InternetFromPort[$port] -eq $testInternet) {
323+
Add-LogMessage -Level Success "The internet is '$($benchmarkVM.InternetFromPort[$port])' on port $port from both"
324+
} else {
325+
Add-LogMessage -Level Failure "The internet is '$($benchmarkVM.InternetFromPort[$port])' on port $port from $($benchmarkVM.Name)"
326+
Add-LogMessage -Level Failure "The internet is '$($testInternet)' on port $port from $($testVM.Name)"
327+
}
328+
}
329+
}
330+
331+
332+
# Switch back to original subscription
333+
# ------------------------------------
334+
$_ = Set-AzContext -Context $originalContext

0 commit comments

Comments
 (0)