USHIFT-2575: add 2 telemetry test cases

test/bin/ci_phase_boot_and_test.sh

@@ -51,12 +51,6 @@ cd "${ROOTDIR}/test"
 # Set up the hypervisor configuration for the tests and start webserver
 bash -x ./bin/manage_hypervisor_config.sh create
 
-# Setup a prometheus server for metrics testing
-bash -x ./bin/manage_prometheus.sh start
-
-# Setup a loki server for metrics testing
-bash -x ./bin/manage_loki.sh start
-
 # Setup a container registry and mirror images.
 bash -x ./bin/mirror_registry.sh
 

test/resources/observability.resource

@@ -0,0 +1,20 @@
+*** Settings ***
+Documentation       Keywords common to observability test suites
+
+
+*** Keywords ***
+Start Prometheus Server
+    [Documentation]    Start a Prometheus Server on port 9092
+    Local Command Should Work    bash -x ./bin/manage_prometheus.sh start
+
+Stop Prometheus Server
+    [Documentation]    Stop the Prometheus Server
+    Local Command Should Work    bash -x ./bin/manage_prometheus.sh stop
+
+Start Loki Server
+    [Documentation]    Start a Loki Server on port 3100
+    Local Command Should Work    bash -x ./bin/manage_loki.sh start
+
+Stop Loki Server
+    [Documentation]    Stop the Loki Server
+    Local Command Should Work    bash -x ./bin/manage_loki.sh stop

test/resources/prometheus.py

@@ -3,7 +3,7 @@
 import libipv6
 
 
-def check_prometheus_query(host: str, port: int, query: str) -> None:
+def _run_prometheus_query(host: str, port: int, query: str) -> requests.Response:
     """Executes the given query against the prometheus instance.
     Fails if the response is empty."""
     base_url = f"http://{host}:{port}/api/v1/query"
@@ -16,10 +16,23 @@ def check_prometheus_query(host: str, port: int, query: str) -> None:
         raise Exception(f"Prometheus query failed with status code {response.status_code}")
     if response.json().get("status") != "success":
         raise Exception(f"Prometheus query failed with status: {response.json().get('status')}")
-    if not response.json().get("data", {}).get("result") or len(response.json().get("data", {}).get("result")) == 0:
+    return response
+
+
+def check_prometheus_query(host: str, port: int, query: str) -> None:
+    response = _run_prometheus_query(host, port, query)
+    data_result_list: list = response.json().get("data", {}).get("result")
+    if not data_result_list or len(data_result_list) == 0:
         raise Exception("Prometheus query returned no results")
 
 
+def check_prometheus_query_is_missing(host: str, port: int, query: str) -> None:
+    response = _run_prometheus_query(host, port, query)
+    data_result_list: list = response.json().get("data", {}).get("result")
+    if data_result_list and len(data_result_list) > 0:
+        raise Exception("Prometheus query returned results")
+
+
 def check_prometheus_exporter(host: str, port: int, query: str) -> None:
     """Check the metric is available int Prometheus Exporter
     Fails if the response is empty."""

test/suites/optional/observability.robot

@@ -10,6 +10,7 @@ Library             ../../resources/loki.py
 Resource            ../../resources/kubeconfig.resource
 Resource            ../../resources/common.resource
 Resource            ../../resources/systemd.resource
+Resource            ../../resources/observability.resource
 
 Suite Setup         Setup Suite And Prepare Test Host
 Suite Teardown      Teardown Suite And Revert Test Host
@@ -31,15 +32,9 @@ Host Metrics Are Exported
     ...    \ [0-9]+, "metrics": [0-9]+, "data points": [0-9]+}
     Pattern Should Appear In Log Output    ${JOURNAL_CUR}    ${pattern}    unit="microshift-observability"
 
-    Check Prometheus Query
-    ...    ${PROMETHEUS_HOST}
-    ...    ${PROMETHEUS_PORT}
-    ...    system_cpu_time_seconds_total{cpu="cpu0",state="idle"}
-
-    Check Prometheus Exporter
-    ...    ${USHIFT_HOST}
-    ...    ${PROM_EXPORTER_PORT}
-    ...    system_cpu_time_seconds_total{cpu="cpu0",state="idle"}
+    Set Test Variable    ${METRIC}    system_cpu_time_seconds_total{cpu="cpu0",state="idle"}
+    Check Prometheus Query    ${PROMETHEUS_HOST}    ${PROMETHEUS_PORT}    ${METRIC}
+    Check Prometheus Exporter    ${USHIFT_HOST}    ${PROM_EXPORTER_PORT}    ${METRIC}
 
 Kube Metrics Are Exported
     [Documentation]    The opentelemetry-collector should be able to export kube metrics.
@@ -49,25 +44,13 @@ Kube Metrics Are Exported
     ...    \ [0-9]+, "metrics": [0-9]+, "data points": [0-9]+}
     Pattern Should Appear In Log Output    ${JOURNAL_CUR}    ${pattern}    unit="microshift-observability"
 
-    Check Prometheus Query
-    ...    ${PROMETHEUS_HOST}
-    ...    ${PROMETHEUS_PORT}
-    ...    container_cpu_time_seconds_total
-
-    Check Prometheus Exporter
-    ...    ${USHIFT_HOST}
-    ...    ${PROM_EXPORTER_PORT}
-    ...    container_cpu_time_seconds_total
-
-    Check Prometheus Query
-    ...    ${PROMETHEUS_HOST}
-    ...    ${PROMETHEUS_PORT}
-    ...    k8s_pod_cpu_time_seconds_total
+    Set Test Variable    ${METRIC}    container_cpu_time_seconds_total
+    Check Prometheus Query    ${PROMETHEUS_HOST}    ${PROMETHEUS_PORT}    ${METRIC}
+    Check Prometheus Exporter    ${USHIFT_HOST}    ${PROM_EXPORTER_PORT}    ${METRIC}
 
-    Check Prometheus Exporter
-    ...    ${USHIFT_HOST}
-    ...    ${PROM_EXPORTER_PORT}
-    ...    k8s_pod_cpu_time_seconds_total
+    Set Test Variable    ${METRIC}    k8s_pod_cpu_time_seconds_total
+    Check Prometheus Query    ${PROMETHEUS_HOST}    ${PROMETHEUS_PORT}    ${METRIC}
+    Check Prometheus Exporter    ${USHIFT_HOST}    ${PROM_EXPORTER_PORT}    ${METRIC}
 
 Journald Logs Are Exported
     [Documentation]    The opentelemetry-collector should be able to export logs to journald.
@@ -78,10 +61,7 @@ Journald Logs Are Exported
     ...    \ "log records": [0-9]+\\}
     Pattern Should Appear In Log Output    ${JOURNAL_CUR}    ${pattern}    unit="microshift-observability"
 
-    Check Loki Query
-    ...    ${LOKI_HOST}
-    ...    ${LOKI_PORT}
-    ...    {job="journald",exporter="OTLP"}
+    Check Loki Query    ${LOKI_HOST}    ${LOKI_PORT}    {job="journald",exporter="OTLP"}
 
 Kube Events Logs Are Exported
     [Documentation]    The opentelemetry-collector should be able to export logs to journald.
@@ -92,10 +72,7 @@ Kube Events Logs Are Exported
     ...    \ "log records": [0-9]+\\}
     Pattern Should Appear In Log Output    ${JOURNAL_CUR}    ${pattern}    unit="microshift-observability"
 
-    Check Loki Query
-    ...    ${LOKI_HOST}
-    ...    ${LOKI_PORT}
-    ...    {job="kube_events",exporter="OTLP"}
+    Check Loki Query    ${LOKI_HOST}    ${LOKI_PORT}    {job="kube_events",exporter="OTLP"}
 
 Logs Should Not Contain Receiver Errors
     [Documentation]    Internal receiver errors are not treated as fatal. Typically these are due to a misconfiguration
@@ -110,6 +87,8 @@ Logs Should Not Contain Receiver Errors
 Setup Suite And Prepare Test Host
     [Documentation]    The service starts after MicroShift starts and thus will start generating pertinent log data
     ...    right away. When the suite is executed, immediately get the cursor for the microshift-observability unit.
+    Start Prometheus Server
+    Start Loki Server
     Setup Suite
     Check Required Observability Variables
     Set Test OTEL Configuration
@@ -153,6 +132,8 @@ Teardown Suite And Revert Test Host
     [Documentation]    Set back original OTEL config and teardown Suite
     Set Back Original OTEL Configuration
     Teardown Suite
+    Stop Loki Server
+    Stop Prometheus Server
 
 Set Back Original OTEL Configuration
     [Documentation]    Set Back Original OTEL Configuration