From b4ebb3e34f4215fb5872138cf03dd51029836c28 Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Wed, 5 Apr 2023 18:18:11 +0200 Subject: [PATCH 01/14] Added prometheus module (wip) --- .gitignore | 1 + lambda_powertools/prometheus.py | 42 +++++++++++++ setup.py | 2 +- tests/test_prometheus.py | 101 ++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 lambda_powertools/prometheus.py create mode 100644 tests/test_prometheus.py diff --git a/.gitignore b/.gitignore index 521ba53..1f9fe3d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ MANIFEST # Unit test / coverage reports .pytest_cache/ +__pycache__/ # Environments .env diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py new file mode 100644 index 0000000..af53b85 --- /dev/null +++ b/lambda_powertools/prometheus.py @@ -0,0 +1,42 @@ +import prometheus_client as client + + +def reset(): + collectors = list(client.REGISTRY._collector_to_names.keys()) + for collector in collectors: + client.REGISTRY.unregister(collector) + + # from prometheus_client import gc_collector, platform_collector, process_collector + # process_collector.ProcessCollector() + # platform_collector.PlatformCollector() + # gc_collector.GCCollector() + + +def filter_metrics(m): + # Other metric types are not supported so far + if not m.type == "counter" and not m.type == "histogram": + return False + + # Empty metrics with labels are exported without values + if len(m.samples) == 0: + return False + + # Empty metrics without labels are exported with zero value and empty labels ¯\_(ツ)_/¯ + if len(m.samples) == 1 and m.samples[0].value == 0 and len(m.samples[0].labels.keys()) == 0: + return False + + return True + + +def get_metrics(): + data = client.REGISTRY.collect() + return list(filter(filter_metrics, data)) + + +def flush_metrics(): + metrics = get_metrics() + + if not metrics or len(metrics) == 0: + return + + print(f'PROMLOG {metrics}') diff --git a/setup.py b/setup.py index 076e50f..d4de806 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ description='Lambda Powertools is a package encapsulating utilities and best practices used to write Python Lambda functions at Spreaker.', author='Spreaker', license='MIT', - install_requires=[], + install_requires=['prometheus-client'], setup_requires=['pytest-runner'], tests_require=['pytest==7.2.2', 'pytest-mock==3.10.0'], test_suite='tests' diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py new file mode 100644 index 0000000..00aa375 --- /dev/null +++ b/tests/test_prometheus.py @@ -0,0 +1,101 @@ +from prometheus_client import Counter, Histogram, Gauge +from lambda_powertools.prometheus import get_metrics, reset + + +counter_no_labels = Counter( + name="prometheus_spec_counter_no_labels", + documentation="Prometheus example counter without labels" +) + +counter_with_labels = Counter( + name="prometheus_spec_counter_with_labels", + documentation="Prometheus example counter with labels", + labelnames=["foo"] +) + +histogram_no_labels = Histogram( + name="prometheus_spec_histogram_no_labels", + documentation="Prometheus example histogram without labels", + buckets=[1, 2, 5] +) + +histogram_with_labels = Histogram( + name="prometheus_spec_histogram_with_labels", + documentation="Prometheus example histogram with labels", + buckets=[1, 2, 5], + labelnames=["foo"] +) + +gauge = Gauge( + name="prometheus_spec_metric_name", + documentation="metric_documentation" +) + + +def test_prometheus_get_metrics_does_not_return_empty_metrics(): + reset() + + metrics = get_metrics() + + assert metrics == [] + + +def test_prometheus_get_metrics_returns_non_empty_metrics(): + reset() + + counter_no_labels.inc(1) + counter_with_labels.labels("bar").inc(2) + histogram_no_labels.observe(1) + histogram_with_labels.labels("bar").observe(1) + gauge.set(1) # Gauge is not supported yet + + metrics = get_metrics() + + assert metrics == [ + { + "documentation": "Prometheus example counter without labels", + "name": "prometheus_spec_counter_no_labels", + "type": "counter", + "samples": [{"value": 1, "labels": {}}], + "aggregator": "sum", + }, + { + "documentation": "Prometheus example counter with labels", + "name": "prometheus_spec_counter_with_labels", + "type": "counter", + "samples": [{"value": 2, "labels": {"foo": "bar"}}], + "aggregator": "sum", + }, + { + "name": "prometheus_spec_histogram_no_labels", + "documentation": "Prometheus example histogram without labels", + "type": "histogram", + "samples": [ + {"labels": {"le": 1}, "value": 1, "name": "prometheus_spec_histogram_no_labels_bucket"}, + {"labels": {"le": 2}, "value": 1, "name": "prometheus_spec_histogram_no_labels_bucket"}, + {"labels": {"le": 5}, "value": 1, "name": "prometheus_spec_histogram_no_labels_bucket"}, + {"labels": {"le": "+Inf"}, "value": 1, "name": "prometheus_spec_histogram_no_labels_bucket"}, + {"labels": {}, "value": 1, "name": "prometheus_spec_histogram_no_labels_sum"}, + {"labels": {}, "value": 1, "name": "prometheus_spec_histogram_no_labels_count"}, + ], + "aggregator": "sum" + }, + { + "name": "prometheus_spec_histogram_with_labels", + "documentation": "Prometheus example histogram with labels", + "type": "histogram", + "samples": [ + {"labels": {"le": 1}, "value": 1, "name": "prometheus_spec_histogram_with_labels_bucket"}, + {"labels": {"le": 2}, "value": 1, "name": "prometheus_spec_histogram_with_labels_bucket"}, + {"labels": {"le": 5}, "value": 1, "name": "prometheus_spec_histogram_with_labels_bucket"}, + { + "labels": {"le": "+Inf"}, + "value": 1, + "name": "prometheus_spec_histogram_with_labels_bucket", + }, + {"labels": {}, "value": 1, "name": "prometheus_spec_histogram_with_labels_sum"}, + {"labels": {}, "value": 1, "name": "prometheus_spec_histogram_with_labels_count"}, + ], + "aggregator": "sum" + } + ] From aa75a88a049b245866e0a67ecd51fb1df55194d7 Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Thu, 6 Apr 2023 12:34:25 +0200 Subject: [PATCH 02/14] Wip --- lambda_powertools/prometheus.py | 5 -- tests/test_prometheus.py | 136 +++++++++++++++----------------- 2 files changed, 63 insertions(+), 78 deletions(-) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index af53b85..f434c96 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -6,11 +6,6 @@ def reset(): for collector in collectors: client.REGISTRY.unregister(collector) - # from prometheus_client import gc_collector, platform_collector, process_collector - # process_collector.ProcessCollector() - # platform_collector.PlatformCollector() - # gc_collector.GCCollector() - def filter_metrics(m): # Other metric types are not supported so far diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py index 00aa375..c2b8f32 100644 --- a/tests/test_prometheus.py +++ b/tests/test_prometheus.py @@ -1,35 +1,50 @@ -from prometheus_client import Counter, Histogram, Gauge +import pytest +import os +import prometheus_client as client +from unittest import mock +from prometheus_client import Counter, Histogram, Gauge, Metric +from prometheus_client.samples import Sample from lambda_powertools.prometheus import get_metrics, reset +counter_no_labels = None +counter_with_labels = None +histogram_no_labels = None +histogram_with_labels = None +gauge = None -counter_no_labels = Counter( - name="prometheus_spec_counter_no_labels", - documentation="Prometheus example counter without labels" -) -counter_with_labels = Counter( - name="prometheus_spec_counter_with_labels", - documentation="Prometheus example counter with labels", - labelnames=["foo"] -) +@pytest.fixture(autouse=True) +def before_each(monkeypatch): + global counter_no_labels, counter_with_labels, histogram_no_labels, histogram_with_labels, gauge -histogram_no_labels = Histogram( - name="prometheus_spec_histogram_no_labels", - documentation="Prometheus example histogram without labels", - buckets=[1, 2, 5] -) + counter_no_labels = Counter( + name="prometheus_spec_counter_no_labels", + documentation="Prometheus example counter without labels" + ) -histogram_with_labels = Histogram( - name="prometheus_spec_histogram_with_labels", - documentation="Prometheus example histogram with labels", - buckets=[1, 2, 5], - labelnames=["foo"] -) + counter_with_labels = Counter( + name="prometheus_spec_counter_with_labels", + documentation="Prometheus example counter with labels", + labelnames=["foo"] + ) -gauge = Gauge( - name="prometheus_spec_metric_name", - documentation="metric_documentation" -) + histogram_no_labels = Histogram( + name="prometheus_spec_histogram_no_labels", + documentation="Prometheus example histogram without labels", + buckets=[1, 2, 5] + ) + + histogram_with_labels = Histogram( + name="prometheus_spec_histogram_with_labels", + documentation="Prometheus example histogram with labels", + buckets=[1, 2, 5], + labelnames=["foo"] + ) + + gauge = Gauge( + name="prometheus_spec_gauge_metric_name", + documentation="gauge_metric_documentation" + ) def test_prometheus_get_metrics_does_not_return_empty_metrics(): @@ -40,8 +55,15 @@ def test_prometheus_get_metrics_does_not_return_empty_metrics(): assert metrics == [] +@mock.patch.dict(os.environ, {"PROMETHEUS_DISABLE_CREATED_SERIES": "True"}) def test_prometheus_get_metrics_returns_non_empty_metrics(): - reset() + # Trying to disable default collector metrics, not under test + try: + client.REGISTRY.unregister(client.GC_COLLECTOR) + client.REGISTRY.unregister(client.PLATFORM_COLLECTOR) + client.REGISTRY.unregister(client.PROCESS_COLLECTOR) + except: + pass counter_no_labels.inc(1) counter_with_labels.labels("bar").inc(2) @@ -51,51 +73,19 @@ def test_prometheus_get_metrics_returns_non_empty_metrics(): metrics = get_metrics() - assert metrics == [ - { - "documentation": "Prometheus example counter without labels", - "name": "prometheus_spec_counter_no_labels", - "type": "counter", - "samples": [{"value": 1, "labels": {}}], - "aggregator": "sum", - }, - { - "documentation": "Prometheus example counter with labels", - "name": "prometheus_spec_counter_with_labels", - "type": "counter", - "samples": [{"value": 2, "labels": {"foo": "bar"}}], - "aggregator": "sum", - }, - { - "name": "prometheus_spec_histogram_no_labels", - "documentation": "Prometheus example histogram without labels", - "type": "histogram", - "samples": [ - {"labels": {"le": 1}, "value": 1, "name": "prometheus_spec_histogram_no_labels_bucket"}, - {"labels": {"le": 2}, "value": 1, "name": "prometheus_spec_histogram_no_labels_bucket"}, - {"labels": {"le": 5}, "value": 1, "name": "prometheus_spec_histogram_no_labels_bucket"}, - {"labels": {"le": "+Inf"}, "value": 1, "name": "prometheus_spec_histogram_no_labels_bucket"}, - {"labels": {}, "value": 1, "name": "prometheus_spec_histogram_no_labels_sum"}, - {"labels": {}, "value": 1, "name": "prometheus_spec_histogram_no_labels_count"}, - ], - "aggregator": "sum" - }, - { - "name": "prometheus_spec_histogram_with_labels", - "documentation": "Prometheus example histogram with labels", - "type": "histogram", - "samples": [ - {"labels": {"le": 1}, "value": 1, "name": "prometheus_spec_histogram_with_labels_bucket"}, - {"labels": {"le": 2}, "value": 1, "name": "prometheus_spec_histogram_with_labels_bucket"}, - {"labels": {"le": 5}, "value": 1, "name": "prometheus_spec_histogram_with_labels_bucket"}, - { - "labels": {"le": "+Inf"}, - "value": 1, - "name": "prometheus_spec_histogram_with_labels_bucket", - }, - {"labels": {}, "value": 1, "name": "prometheus_spec_histogram_with_labels_sum"}, - {"labels": {}, "value": 1, "name": "prometheus_spec_histogram_with_labels_count"}, - ], - "aggregator": "sum" - } + # Remove _created metrics (hack because PROMETHEUS_DISABLE_CREATED_SERIES is not working) + for metric in metrics: + metric.samples = [sample for sample in metric.samples if not sample.name.endswith("_created")] + + expected = [ + Metric("prometheus_spec_counter_no_labels", "Prometheus example counter without labels", "counter", ""), + Metric("prometheus_spec_counter_with_labels", "Prometheus example counter with labels", "counter", ""), + Metric("prometheus_spec_histogram_no_labels", "Prometheus example histogram without labels", "histogram", ""), + Metric("prometheus_spec_histogram_with_labels", "Prometheus example histogram with labels", "histogram", "") ] + expected[0].samples = [Sample(name='prometheus_spec_counter_no_labels_total', labels={}, value=1.0, timestamp=None, exemplar=None)] + expected[1].samples = [Sample(name='prometheus_spec_counter_with_labels_total', labels={'foo': 'bar'}, value=2.0, timestamp=None, exemplar=None)] + expected[2].samples = [Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '1.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '2.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '5.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '+Inf'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_count', labels={}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_sum', labels={}, value=1.0, timestamp=None, exemplar=None)] + expected[3].samples = [Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '1.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '2.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '5.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '+Inf'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_count', labels={'foo': 'bar'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_sum', labels={'foo': 'bar'}, value=1.0, timestamp=None, exemplar=None)] + + assert metrics == expected From 7fdfa48f688a1344cc43d8d8dd9091a1d4f95400 Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Thu, 6 Apr 2023 15:22:37 +0200 Subject: [PATCH 03/14] Fixes --- lambda_powertools/prometheus.py | 11 +++++++++-- tests/test_prometheus.py | 22 +++++++++------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index f434c96..2180b12 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -17,7 +17,7 @@ def filter_metrics(m): return False # Empty metrics without labels are exported with zero value and empty labels ¯\_(ツ)_/¯ - if len(m.samples) == 1 and m.samples[0].value == 0 and len(m.samples[0].labels.keys()) == 0: + if len(m.samples) == 1 and m.samples[0].value == 0.0 and len(m.samples[0].labels.keys()) == 0: return False return True @@ -25,7 +25,14 @@ def filter_metrics(m): def get_metrics(): data = client.REGISTRY.collect() - return list(filter(filter_metrics, data)) + + # Remove _created metrics (hack because PROMETHEUS_DISABLE_CREATED_SERIES is not working) + for metric in data: + metric.samples = [sample for sample in metric.samples if not sample.name.endswith("_created")] + + metrics = list(filter(filter_metrics, data)) + + return metrics def flush_metrics(): diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py index c2b8f32..30615d8 100644 --- a/tests/test_prometheus.py +++ b/tests/test_prometheus.py @@ -14,7 +14,16 @@ @pytest.fixture(autouse=True) +@mock.patch.dict(os.environ, {"PROMETHEUS_DISABLE_CREATED_SERIES": "True"}) def before_each(monkeypatch): + # Trying to disable default collector metrics, not under test + try: + client.REGISTRY.unregister(client.GC_COLLECTOR) + client.REGISTRY.unregister(client.PLATFORM_COLLECTOR) + client.REGISTRY.unregister(client.PROCESS_COLLECTOR) + except: + pass + global counter_no_labels, counter_with_labels, histogram_no_labels, histogram_with_labels, gauge counter_no_labels = Counter( @@ -55,16 +64,7 @@ def test_prometheus_get_metrics_does_not_return_empty_metrics(): assert metrics == [] -@mock.patch.dict(os.environ, {"PROMETHEUS_DISABLE_CREATED_SERIES": "True"}) def test_prometheus_get_metrics_returns_non_empty_metrics(): - # Trying to disable default collector metrics, not under test - try: - client.REGISTRY.unregister(client.GC_COLLECTOR) - client.REGISTRY.unregister(client.PLATFORM_COLLECTOR) - client.REGISTRY.unregister(client.PROCESS_COLLECTOR) - except: - pass - counter_no_labels.inc(1) counter_with_labels.labels("bar").inc(2) histogram_no_labels.observe(1) @@ -73,10 +73,6 @@ def test_prometheus_get_metrics_returns_non_empty_metrics(): metrics = get_metrics() - # Remove _created metrics (hack because PROMETHEUS_DISABLE_CREATED_SERIES is not working) - for metric in metrics: - metric.samples = [sample for sample in metric.samples if not sample.name.endswith("_created")] - expected = [ Metric("prometheus_spec_counter_no_labels", "Prometheus example counter without labels", "counter", ""), Metric("prometheus_spec_counter_with_labels", "Prometheus example counter with labels", "counter", ""), From 7ae51d595120455157f8c09ad1d4db1f20b1e9d8 Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Thu, 6 Apr 2023 16:12:56 +0200 Subject: [PATCH 04/14] Added prometheus to the runtime --- lambda_powertools/metric_encoder.py | 15 ++++++++ lambda_powertools/prometheus.py | 6 ++- lambda_powertools/runtime.py | 5 +++ tests/test_prometheus.py | 58 ++++++++++++++--------------- 4 files changed, 54 insertions(+), 30 deletions(-) create mode 100644 lambda_powertools/metric_encoder.py diff --git a/lambda_powertools/metric_encoder.py b/lambda_powertools/metric_encoder.py new file mode 100644 index 0000000..f5eb182 --- /dev/null +++ b/lambda_powertools/metric_encoder.py @@ -0,0 +1,15 @@ +import json +from prometheus_client import Metric + + +class MetricEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Metric): + return { + "help": obj.documentation, + "name": obj.name, + "type": obj.type, + "values": json.dumps(obj.samples), + "aggregator": "sum", + } + return json.JSONEncoder.default(self, obj) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index 2180b12..c1171d9 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -1,4 +1,6 @@ +import json import prometheus_client as client +from lambda_powertools.metric_encoder import MetricEncoder def reset(): @@ -41,4 +43,6 @@ def flush_metrics(): if not metrics or len(metrics) == 0: return - print(f'PROMLOG {metrics}') + json_metrics = json.dumps(metrics, cls=MetricEncoder) + + print(f'PROMLOG {json_metrics}') diff --git a/lambda_powertools/runtime.py b/lambda_powertools/runtime.py index 4bbcfc7..7ce4d5b 100644 --- a/lambda_powertools/runtime.py +++ b/lambda_powertools/runtime.py @@ -1,4 +1,5 @@ import os +import lambda_powertools.prometheus as prometheus from lambda_powertools.logger import logger @@ -8,6 +9,8 @@ def wrapper(event, context): logger.reset() logger.capture(env=os.environ, event=event, context=context) + prometheus.reset() + response = None error = None try: @@ -15,6 +18,8 @@ def wrapper(event, context): except Exception as e: error = e + prometheus.flush_metrics() + if error is not None: raise error diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py index 30615d8..b0953e0 100644 --- a/tests/test_prometheus.py +++ b/tests/test_prometheus.py @@ -56,32 +56,32 @@ def before_each(monkeypatch): ) -def test_prometheus_get_metrics_does_not_return_empty_metrics(): - reset() - - metrics = get_metrics() - - assert metrics == [] - - -def test_prometheus_get_metrics_returns_non_empty_metrics(): - counter_no_labels.inc(1) - counter_with_labels.labels("bar").inc(2) - histogram_no_labels.observe(1) - histogram_with_labels.labels("bar").observe(1) - gauge.set(1) # Gauge is not supported yet - - metrics = get_metrics() - - expected = [ - Metric("prometheus_spec_counter_no_labels", "Prometheus example counter without labels", "counter", ""), - Metric("prometheus_spec_counter_with_labels", "Prometheus example counter with labels", "counter", ""), - Metric("prometheus_spec_histogram_no_labels", "Prometheus example histogram without labels", "histogram", ""), - Metric("prometheus_spec_histogram_with_labels", "Prometheus example histogram with labels", "histogram", "") - ] - expected[0].samples = [Sample(name='prometheus_spec_counter_no_labels_total', labels={}, value=1.0, timestamp=None, exemplar=None)] - expected[1].samples = [Sample(name='prometheus_spec_counter_with_labels_total', labels={'foo': 'bar'}, value=2.0, timestamp=None, exemplar=None)] - expected[2].samples = [Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '1.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '2.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '5.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '+Inf'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_count', labels={}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_sum', labels={}, value=1.0, timestamp=None, exemplar=None)] - expected[3].samples = [Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '1.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '2.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '5.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '+Inf'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_count', labels={'foo': 'bar'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_sum', labels={'foo': 'bar'}, value=1.0, timestamp=None, exemplar=None)] - - assert metrics == expected +# def test_prometheus_get_metrics_does_not_return_empty_metrics(): +# reset() +# +# metrics = get_metrics() +# +# assert metrics == [] + + +# def test_prometheus_get_metrics_returns_non_empty_metrics(): +# counter_no_labels.inc(1) +# counter_with_labels.labels("bar").inc(2) +# histogram_no_labels.observe(1) +# histogram_with_labels.labels("bar").observe(1) +# gauge.set(1) # Gauge is not supported yet +# +# metrics = get_metrics() +# +# expected = [ +# Metric("prometheus_spec_counter_no_labels", "Prometheus example counter without labels", "counter", ""), +# Metric("prometheus_spec_counter_with_labels", "Prometheus example counter with labels", "counter", ""), +# Metric("prometheus_spec_histogram_no_labels", "Prometheus example histogram without labels", "histogram", ""), +# Metric("prometheus_spec_histogram_with_labels", "Prometheus example histogram with labels", "histogram", "") +# ] +# expected[0].samples = [Sample(name='prometheus_spec_counter_no_labels_total', labels={}, value=1.0, timestamp=None, exemplar=None)] +# expected[1].samples = [Sample(name='prometheus_spec_counter_with_labels_total', labels={'foo': 'bar'}, value=2.0, timestamp=None, exemplar=None)] +# expected[2].samples = [Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '1.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '2.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '5.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '+Inf'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_count', labels={}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_sum', labels={}, value=1.0, timestamp=None, exemplar=None)] +# expected[3].samples = [Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '1.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '2.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '5.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '+Inf'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_count', labels={'foo': 'bar'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_sum', labels={'foo': 'bar'}, value=1.0, timestamp=None, exemplar=None)] +# +# assert metrics == expected From 07bd2cebfb1c83202cfd6caaa5607a1aa054c9ec Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Wed, 8 Nov 2023 17:32:02 +0100 Subject: [PATCH 05/14] wip --- lambda_powertools/metric_encoder.py | 15 --- lambda_powertools/prometheus.py | 61 +++++++++---- tests/test_prometheus.py | 137 ++++++++++++---------------- 3 files changed, 97 insertions(+), 116 deletions(-) delete mode 100644 lambda_powertools/metric_encoder.py diff --git a/lambda_powertools/metric_encoder.py b/lambda_powertools/metric_encoder.py deleted file mode 100644 index f5eb182..0000000 --- a/lambda_powertools/metric_encoder.py +++ /dev/null @@ -1,15 +0,0 @@ -import json -from prometheus_client import Metric - - -class MetricEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, Metric): - return { - "help": obj.documentation, - "name": obj.name, - "type": obj.type, - "values": json.dumps(obj.samples), - "aggregator": "sum", - } - return json.JSONEncoder.default(self, obj) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index c1171d9..814bbb6 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -1,40 +1,63 @@ +import os import json -import prometheus_client as client -from lambda_powertools.metric_encoder import MetricEncoder + +os.environ['PROMETHEUS_DISABLE_CREATED_SERIES'] = 'True' +import prometheus_client +from prometheus_client import CollectorRegistry + +prometheus_client.REGISTRY.unregister(prometheus_client.GC_COLLECTOR) +prometheus_client.REGISTRY.unregister(prometheus_client.PLATFORM_COLLECTOR) +prometheus_client.REGISTRY.unregister(prometheus_client.PROCESS_COLLECTOR) def reset(): - collectors = list(client.REGISTRY._collector_to_names.keys()) - for collector in collectors: - client.REGISTRY.unregister(collector) + for collector in list(prometheus_client.REGISTRY._collector_to_names.keys()): + if "_value" in dir(collector): + # case Counter without labels + collector._value.set(0) + elif "_metrics" in dir(collector): + for metric in collector._metrics.values(): + if "_value" in dir(metric): + # case Counter with labels + metric._value.set(0) + elif "_buckets" in dir(metric) and "_sum" in dir(metric): + # case Histogram with labels + metric._sum.set(0) + for bucket in metric._buckets: + bucket.set(0) + elif "_buckets" in dir(collector) and "_sum" in dir(collector): + # case Histogram without labels + collector._sum.set(0) + for bucket in collector._buckets: + bucket.set(0) def filter_metrics(m): # Other metric types are not supported so far - if not m.type == "counter" and not m.type == "histogram": + if not m._type == "counter" and not m._type == "histogram": return False # Empty metrics with labels are exported without values - if len(m.samples) == 0: + dir_m = dir(m) + if "_value" in dir_m and m._value._value == 0: return False - - # Empty metrics without labels are exported with zero value and empty labels ¯\_(ツ)_/¯ - if len(m.samples) == 1 and m.samples[0].value == 0.0 and len(m.samples[0].labels.keys()) == 0: + if "_metrics" in dir_m and len(m._metrics.values()) == 0: + return False + if "_buckets" in dir_m and m._sum._value == 0: return False return True def get_metrics(): - data = client.REGISTRY.collect() - - # Remove _created metrics (hack because PROMETHEUS_DISABLE_CREATED_SERIES is not working) - for metric in data: - metric.samples = [sample for sample in metric.samples if not sample.name.endswith("_created")] + new_registry = CollectorRegistry(auto_describe=True) + for collector in list(prometheus_client.REGISTRY._collector_to_names.keys()): + if (filter_metrics(collector)): + new_registry.register(collector) - metrics = list(filter(filter_metrics, data)) + data = prometheus_client.generate_latest(new_registry).decode("utf-8") - return metrics + return data def flush_metrics(): @@ -43,6 +66,4 @@ def flush_metrics(): if not metrics or len(metrics) == 0: return - json_metrics = json.dumps(metrics, cls=MetricEncoder) - - print(f'PROMLOG {json_metrics}') + print("PROMLOG [" + json.dumps(metrics) + "]") diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py index b0953e0..388904c 100644 --- a/tests/test_prometheus.py +++ b/tests/test_prometheus.py @@ -1,87 +1,62 @@ import pytest -import os -import prometheus_client as client -from unittest import mock -from prometheus_client import Counter, Histogram, Gauge, Metric -from prometheus_client.samples import Sample from lambda_powertools.prometheus import get_metrics, reset - -counter_no_labels = None -counter_with_labels = None -histogram_no_labels = None -histogram_with_labels = None -gauge = None +from prometheus_client import Counter, Histogram, Gauge + +counter_no_labels = Counter( + name="prometheus_spec_counter_no_labels", + documentation="Prometheus example counter without labels" +) + +counter_with_labels = Counter( + name="prometheus_spec_counter_with_labels", + documentation="Prometheus example counter with labels", + labelnames=["foo"] +) + +histogram_no_labels = Histogram( + name="prometheus_spec_histogram_no_labels", + documentation="Prometheus example histogram without labels", + buckets=[1, 2, 5] +) + +histogram_with_labels = Histogram( + name="prometheus_spec_histogram_with_labels", + documentation="Prometheus example histogram with labels", + buckets=[1, 2, 5], + labelnames=["foo"] +) + +gauge = Gauge( + name="prometheus_spec_gauge_metric_name", + documentation="gauge_metric_documentation" +) @pytest.fixture(autouse=True) -@mock.patch.dict(os.environ, {"PROMETHEUS_DISABLE_CREATED_SERIES": "True"}) def before_each(monkeypatch): - # Trying to disable default collector metrics, not under test - try: - client.REGISTRY.unregister(client.GC_COLLECTOR) - client.REGISTRY.unregister(client.PLATFORM_COLLECTOR) - client.REGISTRY.unregister(client.PROCESS_COLLECTOR) - except: - pass - - global counter_no_labels, counter_with_labels, histogram_no_labels, histogram_with_labels, gauge - - counter_no_labels = Counter( - name="prometheus_spec_counter_no_labels", - documentation="Prometheus example counter without labels" - ) - - counter_with_labels = Counter( - name="prometheus_spec_counter_with_labels", - documentation="Prometheus example counter with labels", - labelnames=["foo"] - ) - - histogram_no_labels = Histogram( - name="prometheus_spec_histogram_no_labels", - documentation="Prometheus example histogram without labels", - buckets=[1, 2, 5] - ) - - histogram_with_labels = Histogram( - name="prometheus_spec_histogram_with_labels", - documentation="Prometheus example histogram with labels", - buckets=[1, 2, 5], - labelnames=["foo"] - ) - - gauge = Gauge( - name="prometheus_spec_gauge_metric_name", - documentation="gauge_metric_documentation" - ) - - -# def test_prometheus_get_metrics_does_not_return_empty_metrics(): -# reset() -# -# metrics = get_metrics() -# -# assert metrics == [] - - -# def test_prometheus_get_metrics_returns_non_empty_metrics(): -# counter_no_labels.inc(1) -# counter_with_labels.labels("bar").inc(2) -# histogram_no_labels.observe(1) -# histogram_with_labels.labels("bar").observe(1) -# gauge.set(1) # Gauge is not supported yet -# -# metrics = get_metrics() -# -# expected = [ -# Metric("prometheus_spec_counter_no_labels", "Prometheus example counter without labels", "counter", ""), -# Metric("prometheus_spec_counter_with_labels", "Prometheus example counter with labels", "counter", ""), -# Metric("prometheus_spec_histogram_no_labels", "Prometheus example histogram without labels", "histogram", ""), -# Metric("prometheus_spec_histogram_with_labels", "Prometheus example histogram with labels", "histogram", "") -# ] -# expected[0].samples = [Sample(name='prometheus_spec_counter_no_labels_total', labels={}, value=1.0, timestamp=None, exemplar=None)] -# expected[1].samples = [Sample(name='prometheus_spec_counter_with_labels_total', labels={'foo': 'bar'}, value=2.0, timestamp=None, exemplar=None)] -# expected[2].samples = [Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '1.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '2.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '5.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_bucket', labels={'le': '+Inf'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_count', labels={}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_no_labels_sum', labels={}, value=1.0, timestamp=None, exemplar=None)] -# expected[3].samples = [Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '1.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '2.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '5.0'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_bucket', labels={'foo': 'bar', 'le': '+Inf'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_count', labels={'foo': 'bar'}, value=1.0, timestamp=None, exemplar=None), Sample(name='prometheus_spec_histogram_with_labels_sum', labels={'foo': 'bar'}, value=1.0, timestamp=None, exemplar=None)] -# -# assert metrics == expected + reset() + + +def test_prometheus_get_metrics_does_not_return_empty_metrics(): + reset() + + metrics = get_metrics() + + assert metrics == "" + + +def test_prometheus_get_metrics_returns_non_empty_metrics(): + gauge.set(1) # Gauge is not supported yet + counter_no_labels.inc(1) + # counter_with_labels.labels("bar").inc(2) + # histogram_no_labels.observe(1) + # histogram_with_labels.labels("bar").observe(1) + + metrics = get_metrics() + + expected = """# HELP prometheus_spec_counter_no_labels_total Prometheus example counter without labels +# TYPE prometheus_spec_counter_no_labels_total counter +prometheus_spec_counter_no_labels_total 1.0 +""" + + assert metrics == expected From d10ab2b28599649b97aed4e3d198a298007183aa Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Thu, 9 Nov 2023 11:21:01 +0100 Subject: [PATCH 06/14] wip --- lambda_powertools/prometheus.py | 8 +++----- tests/test_prometheus.py | 31 ++++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index 814bbb6..39355dd 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -51,13 +51,11 @@ def filter_metrics(m): def get_metrics(): new_registry = CollectorRegistry(auto_describe=True) - for collector in list(prometheus_client.REGISTRY._collector_to_names.keys()): - if (filter_metrics(collector)): - new_registry.register(collector) - data = prometheus_client.generate_latest(new_registry).decode("utf-8") + for collector in list(filter(filter_metrics, prometheus_client.REGISTRY._collector_to_names.keys())): + new_registry.register(collector) - return data + return prometheus_client.generate_latest(new_registry).decode("utf-8") def flush_metrics(): diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py index 388904c..994cf14 100644 --- a/tests/test_prometheus.py +++ b/tests/test_prometheus.py @@ -7,6 +7,11 @@ documentation="Prometheus example counter without labels" ) +counter_no_labels_no_value = Counter( + name="prometheus_spec_counter_no_labels_no_value", + documentation="Prometheus example counter without labels" +) + counter_with_labels = Counter( name="prometheus_spec_counter_with_labels", documentation="Prometheus example counter with labels", @@ -48,15 +53,35 @@ def test_prometheus_get_metrics_does_not_return_empty_metrics(): def test_prometheus_get_metrics_returns_non_empty_metrics(): gauge.set(1) # Gauge is not supported yet counter_no_labels.inc(1) - # counter_with_labels.labels("bar").inc(2) - # histogram_no_labels.observe(1) - # histogram_with_labels.labels("bar").observe(1) + counter_with_labels.labels("bar").inc(2) + histogram_no_labels.observe(2) + histogram_no_labels.observe(5) + histogram_with_labels.labels("bar").observe(2) metrics = get_metrics() expected = """# HELP prometheus_spec_counter_no_labels_total Prometheus example counter without labels # TYPE prometheus_spec_counter_no_labels_total counter prometheus_spec_counter_no_labels_total 1.0 +# HELP prometheus_spec_counter_with_labels_total Prometheus example counter with labels +# TYPE prometheus_spec_counter_with_labels_total counter +prometheus_spec_counter_with_labels_total{foo="bar"} 2.0 +# HELP prometheus_spec_histogram_no_labels Prometheus example histogram without labels +# TYPE prometheus_spec_histogram_no_labels histogram +prometheus_spec_histogram_no_labels_bucket{le="1.0"} 0.0 +prometheus_spec_histogram_no_labels_bucket{le="2.0"} 1.0 +prometheus_spec_histogram_no_labels_bucket{le="5.0"} 2.0 +prometheus_spec_histogram_no_labels_bucket{le="+Inf"} 2.0 +prometheus_spec_histogram_no_labels_count 2.0 +prometheus_spec_histogram_no_labels_sum 7.0 +# HELP prometheus_spec_histogram_with_labels Prometheus example histogram with labels +# TYPE prometheus_spec_histogram_with_labels histogram +prometheus_spec_histogram_with_labels_bucket{foo="bar",le="1.0"} 0.0 +prometheus_spec_histogram_with_labels_bucket{foo="bar",le="2.0"} 1.0 +prometheus_spec_histogram_with_labels_bucket{foo="bar",le="5.0"} 1.0 +prometheus_spec_histogram_with_labels_bucket{foo="bar",le="+Inf"} 1.0 +prometheus_spec_histogram_with_labels_count{foo="bar"} 1.0 +prometheus_spec_histogram_with_labels_sum{foo="bar"} 2.0 """ assert metrics == expected From ef249a6e55b08deb49895d385cf222c79816d7ca Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Thu, 9 Nov 2023 11:33:54 +0100 Subject: [PATCH 07/14] Added check on module import order --- lambda_powertools/prometheus.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index 39355dd..bd2df8d 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -1,5 +1,10 @@ import os import json +import sys + +modulename = 'prometheus_client' +if modulename in sys.modules: + raise Exception("prometheus_client already imported, lambda_powertools.prometheus must be imported before") os.environ['PROMETHEUS_DISABLE_CREATED_SERIES'] = 'True' import prometheus_client From 320f312153769cf3abd23f72ebf41368f2d1ca91 Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Thu, 9 Nov 2023 11:50:35 +0100 Subject: [PATCH 08/14] Fix --- lambda_powertools/prometheus.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index bd2df8d..eb93883 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -69,4 +69,7 @@ def flush_metrics(): if not metrics or len(metrics) == 0: return + if os.environ.get("PYTEST_CURRENT_TEST"): + return + print("PROMLOG [" + json.dumps(metrics) + "]") From 67f343228cddd4680b8c8f69ba2e0a1d322c390d Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Thu, 9 Nov 2023 12:54:24 +0100 Subject: [PATCH 09/14] Fix --- lambda_powertools/prometheus.py | 7 +++++-- tests/test_prometheus.py | 23 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index eb93883..406fd50 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -46,10 +46,13 @@ def filter_metrics(m): dir_m = dir(m) if "_value" in dir_m and m._value._value == 0: return False - if "_metrics" in dir_m and len(m._metrics.values()) == 0: - return False if "_buckets" in dir_m and m._sum._value == 0: return False + if "_metrics" in dir_m: + for metric in m._metrics.values(): + if ("_value" in dir(metric) and metric._value._value > 0) or ("_sum" in dir(metric) and metric._sum._value > 0): + return True + return False return True diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py index 994cf14..4a763bd 100644 --- a/tests/test_prometheus.py +++ b/tests/test_prometheus.py @@ -1,5 +1,8 @@ +import os import pytest -from lambda_powertools.prometheus import get_metrics, reset +from unittest.mock import patch + +from lambda_powertools.prometheus import get_metrics, reset, flush_metrics from prometheus_client import Counter, Histogram, Gauge counter_no_labels = Counter( @@ -43,6 +46,13 @@ def before_each(monkeypatch): def test_prometheus_get_metrics_does_not_return_empty_metrics(): + gauge.set(1) # Gauge is not supported yet + counter_no_labels.inc(1) + counter_with_labels.labels("bar").inc(2) + histogram_no_labels.observe(2) + histogram_no_labels.observe(5) + histogram_with_labels.labels("bar").observe(2) + reset() metrics = get_metrics() @@ -85,3 +95,14 @@ def test_prometheus_get_metrics_returns_non_empty_metrics(): """ assert metrics == expected + + +@patch('builtins.print') +@patch.dict(os.environ, {"PYTEST_CURRENT_TEST": ""}) +def test_prometheus_flush_metrics(mock_print): + counter_no_labels.inc(1) + counter_with_labels.labels("bar").inc(2) + + flush_metrics() + + mock_print.assert_called_with('PROMLOG ["# HELP prometheus_spec_counter_no_labels_total Prometheus example counter without labels\\n# TYPE prometheus_spec_counter_no_labels_total counter\\nprometheus_spec_counter_no_labels_total 1.0\\n# HELP prometheus_spec_counter_with_labels_total Prometheus example counter with labels\\n# TYPE prometheus_spec_counter_with_labels_total counter\\nprometheus_spec_counter_with_labels_total{foo=\\"bar\\"} 2.0\\n"]') From 08740f7eb82584442bc235e72afd0bc5d1ef2840 Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Fri, 10 Nov 2023 14:25:07 +0100 Subject: [PATCH 10/14] Testing broken test with pi --- tests/test_prometheus.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py index 4a763bd..2a958b0 100644 --- a/tests/test_prometheus.py +++ b/tests/test_prometheus.py @@ -73,9 +73,6 @@ def test_prometheus_get_metrics_returns_non_empty_metrics(): expected = """# HELP prometheus_spec_counter_no_labels_total Prometheus example counter without labels # TYPE prometheus_spec_counter_no_labels_total counter prometheus_spec_counter_no_labels_total 1.0 -# HELP prometheus_spec_counter_with_labels_total Prometheus example counter with labels -# TYPE prometheus_spec_counter_with_labels_total counter -prometheus_spec_counter_with_labels_total{foo="bar"} 2.0 # HELP prometheus_spec_histogram_no_labels Prometheus example histogram without labels # TYPE prometheus_spec_histogram_no_labels histogram prometheus_spec_histogram_no_labels_bucket{le="1.0"} 0.0 From 19713edd830000da31f17a22465a1f2de3f91615 Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Fri, 10 Nov 2023 14:41:05 +0100 Subject: [PATCH 11/14] Revert "Testing broken test with pi" This reverts commit 08740f7eb82584442bc235e72afd0bc5d1ef2840. --- tests/test_prometheus.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py index 2a958b0..4a763bd 100644 --- a/tests/test_prometheus.py +++ b/tests/test_prometheus.py @@ -73,6 +73,9 @@ def test_prometheus_get_metrics_returns_non_empty_metrics(): expected = """# HELP prometheus_spec_counter_no_labels_total Prometheus example counter without labels # TYPE prometheus_spec_counter_no_labels_total counter prometheus_spec_counter_no_labels_total 1.0 +# HELP prometheus_spec_counter_with_labels_total Prometheus example counter with labels +# TYPE prometheus_spec_counter_with_labels_total counter +prometheus_spec_counter_with_labels_total{foo="bar"} 2.0 # HELP prometheus_spec_histogram_no_labels Prometheus example histogram without labels # TYPE prometheus_spec_histogram_no_labels histogram prometheus_spec_histogram_no_labels_bucket{le="1.0"} 0.0 From 87fe9aed97dbd273ff91f15c35d1bfa60573bf73 Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Fri, 10 Nov 2023 17:23:11 +0100 Subject: [PATCH 12/14] Improvements --- lambda_powertools/prometheus.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index 406fd50..a413986 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -2,11 +2,10 @@ import json import sys -modulename = 'prometheus_client' -if modulename in sys.modules: +if "prometheus_client" in sys.modules: raise Exception("prometheus_client already imported, lambda_powertools.prometheus must be imported before") -os.environ['PROMETHEUS_DISABLE_CREATED_SERIES'] = 'True' +os.environ["PROMETHEUS_DISABLE_CREATED_SERIES"] = "True" import prometheus_client from prometheus_client import CollectorRegistry @@ -16,7 +15,7 @@ def reset(): - for collector in list(prometheus_client.REGISTRY._collector_to_names.keys()): + for collector in prometheus_client.REGISTRY._collector_to_names: if "_value" in dir(collector): # case Counter without labels collector._value.set(0) @@ -39,7 +38,7 @@ def reset(): def filter_metrics(m): # Other metric types are not supported so far - if not m._type == "counter" and not m._type == "histogram": + if m._type not in ("counter", "histogram"): return False # Empty metrics with labels are exported without values @@ -60,7 +59,7 @@ def filter_metrics(m): def get_metrics(): new_registry = CollectorRegistry(auto_describe=True) - for collector in list(filter(filter_metrics, prometheus_client.REGISTRY._collector_to_names.keys())): + for collector in filter(filter_metrics, prometheus_client.REGISTRY._collector_to_names): new_registry.register(collector) return prometheus_client.generate_latest(new_registry).decode("utf-8") From afb8dc92ec246b50f4daeea773a485c4e782c39c Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Fri, 10 Nov 2023 17:29:55 +0100 Subject: [PATCH 13/14] Prometheus-client 0.18.0; updated version to 0.2.0 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d4de806..6cc687d 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,11 @@ setup( name='lambda_powertools', packages=find_packages(include=['lambda_powertools']), - version='0.1.0', + version='0.2.0', description='Lambda Powertools is a package encapsulating utilities and best practices used to write Python Lambda functions at Spreaker.', author='Spreaker', license='MIT', - install_requires=['prometheus-client'], + install_requires=['prometheus-client==0.18.0'], setup_requires=['pytest-runner'], tests_require=['pytest==7.2.2', 'pytest-mock==3.10.0'], test_suite='tests' From eb24722441e0631a6d63e5f8cca0d10e24baf346 Mon Sep 17 00:00:00 2001 From: Mauro Piccotti Date: Fri, 10 Nov 2023 18:03:17 +0100 Subject: [PATCH 14/14] Used the disable_created_metrics() from prometheus client 0.18 --- lambda_powertools/prometheus.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lambda_powertools/prometheus.py b/lambda_powertools/prometheus.py index a413986..906bdde 100644 --- a/lambda_powertools/prometheus.py +++ b/lambda_powertools/prometheus.py @@ -1,14 +1,9 @@ import os import json -import sys - -if "prometheus_client" in sys.modules: - raise Exception("prometheus_client already imported, lambda_powertools.prometheus must be imported before") - -os.environ["PROMETHEUS_DISABLE_CREATED_SERIES"] = "True" import prometheus_client from prometheus_client import CollectorRegistry +prometheus_client.disable_created_metrics() prometheus_client.REGISTRY.unregister(prometheus_client.GC_COLLECTOR) prometheus_client.REGISTRY.unregister(prometheus_client.PLATFORM_COLLECTOR) prometheus_client.REGISTRY.unregister(prometheus_client.PROCESS_COLLECTOR)