Skip to content

Commit 5287b0f

Browse files
ArthurSensroboquat
authored andcommitted
Wait for ingress readiness before completing werft job
Signed-off-by: ArthurSens <[email protected]>
1 parent 4425adc commit 5287b0f

File tree

2 files changed

+41
-1
lines changed

2 files changed

+41
-1
lines changed

.werft/util/observability.ts

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { werft, exec } from './shell';
22
import * as shell from 'shelljs';
33
import * as fs from 'fs';
4+
import { validateIPaddress } from './util';
45

56
/**
67
* Monitoring satellite deployment bits
@@ -52,6 +53,7 @@ export async function installMonitoringSatellite(params: InstallMonitoringSatell
5253
exec(jsonnetRenderCmd, {silent: true})
5354
// The correct kubectl context should already be configured prior to this step
5455
ensureCorrectInstallationOrder()
56+
ensureIngressesReadiness(params)
5557
}
5658

5759
async function ensureCorrectInstallationOrder(){
@@ -152,3 +154,34 @@ function jsonnetUnitTests(): boolean {
152154
}
153155
return success
154156
}
157+
158+
function ensureIngressesReadiness(params: InstallMonitoringSatelliteParams) {
159+
// Read more about validating ingresses readiness
160+
// https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balance-ingress?hl=it#validate
161+
162+
let grafanaIngressReady = false
163+
let prometheusIngressReady = false
164+
werft.log(sliceName, "Checking ingresses readiness")
165+
for(let i = 0; i < 15; i++) {
166+
grafanaIngressReady = ingressReady(params.satelliteNamespace, 'grafana')
167+
prometheusIngressReady = ingressReady(params.satelliteNamespace, 'prometheus')
168+
169+
if(grafanaIngressReady && prometheusIngressReady) { break }
170+
werft.log(sliceName, "Trying again in 1 minute")
171+
exec(`sleep 60`, {slice: sliceName}) // 1 min
172+
i++
173+
}
174+
175+
if (!prometheusIngressReady || !grafanaIngressReady) {
176+
throw new Error('Timeout while waiting for ingresses readiness')
177+
}
178+
}
179+
180+
function ingressReady(namespace: string, name: string): boolean {
181+
let ingressAddress = exec(`kubectl get ingress -n ${namespace} --no-headers ${name} | awk {'print $4'}`, {silent: true}).stdout.trim()
182+
if (validateIPaddress(ingressAddress)) {
183+
return true
184+
}
185+
werft.log(sliceName, `${name} ingress not ready.`)
186+
return false
187+
}

.werft/util/util.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,11 @@ export async function sleep(millis: number) {
22
return new Promise((resolve) => {
33
setTimeout(resolve, millis);
44
});
5-
}
5+
}
6+
7+
export function validateIPaddress(ipaddress) {
8+
if (/^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/.test(ipaddress)) {
9+
return true
10+
}
11+
return false
12+
}

0 commit comments

Comments
 (0)