From 0244fd2c2ba365f09f366e61e38cba3c571e1dcc Mon Sep 17 00:00:00 2001 From: Daniel Roschka Date: Fri, 28 Aug 2020 14:59:28 +0200 Subject: [PATCH 1/2] Add a DISABLE_METRIC_EXTRACTION config option This adds a new configuration option `DISABLE_METRIC_EXTRACTION` which removes the `_aws` metadata from the EMF log records, thus disabling the extraction of metrics by CloudWatch Metrics. The remaining part of the log records is still getting logged. --- README.md | 15 +++++++++++ aws_embedded_metrics/config/configuration.py | 2 ++ .../environment_configuration_provider.py | 2 ++ .../serializers/log_serializer.py | 25 +++++++++++++------ tests/config/test_config.py | 7 ++++++ tests/serializer/test_log_serializer.py | 22 ++++++++++++++++ tests/sinks/test_lambda_sink.py | 5 ++++ 7 files changed, 71 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3bd1fd7..58af6a7 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,21 @@ Config.namespace = "MyApplication" AWS_EMF_NAMESPACE = MyApplication ``` +**DISABLE_METRIC_EXTRACTION**: Disables extraction of metrics by CloudWatch, by omitting EMF +metadata from serialized log records. + +Example: + +```py +# in process +from aws_embedded_metrics.config import get_config +Config = get_config() +Config.disable_metric_extraction = True + +# environment +AWS_EMF_DISABLE_METRIC_EXTRACTION = true +``` + ## Examples Check out the [examples](https://github.com/awslabs/aws-embedded-metrics-python/tree/master/examples) directory to get started. diff --git a/aws_embedded_metrics/config/configuration.py b/aws_embedded_metrics/config/configuration.py index d2aecc8..8628e26 100644 --- a/aws_embedded_metrics/config/configuration.py +++ b/aws_embedded_metrics/config/configuration.py @@ -23,6 +23,7 @@ def __init__( agent_endpoint: str, ec2_metadata_endpoint: str = None, namespace: str = None, + disable_metric_extraction: bool = False, ): self.debug_logging_enabled = debug_logging_enabled self.service_name = service_name @@ -32,3 +33,4 @@ def __init__( self.agent_endpoint = agent_endpoint self.ec2_metadata_endpoint = ec2_metadata_endpoint self.namespace = namespace + self.disable_metric_extraction = disable_metric_extraction diff --git a/aws_embedded_metrics/config/environment_configuration_provider.py b/aws_embedded_metrics/config/environment_configuration_provider.py index 17c5afc..f9102ae 100644 --- a/aws_embedded_metrics/config/environment_configuration_provider.py +++ b/aws_embedded_metrics/config/environment_configuration_provider.py @@ -24,6 +24,7 @@ AGENT_ENDPOINT = "AGENT_ENDPOINT" EC2_METADATA_ENDPOINT = "EC2_METADATA_ENDPOINT" NAMESPACE = "NAMESPACE" +DISABLE_METRIC_EXTRACTION = "DISABLE_METRIC_EXTRACTION" class EnvironmentConfigurationProvider: @@ -41,6 +42,7 @@ def get_configuration(self) -> Configuration: self.__get_env_var(AGENT_ENDPOINT), self.__get_env_var(EC2_METADATA_ENDPOINT), self.__get_env_var(NAMESPACE), + self.__get_bool_env_var(DISABLE_METRIC_EXTRACTION), ) @staticmethod diff --git a/aws_embedded_metrics/serializers/log_serializer.py b/aws_embedded_metrics/serializers/log_serializer.py index afb5f73..6aa4176 100644 --- a/aws_embedded_metrics/serializers/log_serializer.py +++ b/aws_embedded_metrics/serializers/log_serializer.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from aws_embedded_metrics.config import get_config from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.serializers import Serializer from aws_embedded_metrics.constants import MAX_DIMENSIONS, MAX_METRICS_PER_EVENT @@ -18,9 +19,12 @@ from typing import Any, Dict, List + class LogSerializer(Serializer): @staticmethod def serialize(context: MetricsContext) -> List[str]: + config = get_config() + dimension_keys = [] dimensions_properties: Dict[str, str] = {} @@ -30,10 +34,12 @@ def serialize(context: MetricsContext) -> List[str]: dimensions_properties = {**dimensions_properties, **dimension_set} def create_body() -> Dict[str, Any]: - return { + body = { **dimensions_properties, **context.properties, - "_aws": { + } + if not config.disable_metric_extraction: + body["_aws"] = { **context.meta, "CloudWatchMetrics": [ { @@ -42,11 +48,12 @@ def create_body() -> Dict[str, Any]: "Namespace": context.namespace, }, ], - }, - } + } + return body current_body: Dict[str, Any] = create_body() event_batches: List[str] = [] + num_metrics_in_current_body = 0 for metric_name, metric in context.metrics.items(): @@ -55,14 +62,18 @@ def create_body() -> Dict[str, Any]: else: current_body[metric_name] = metric.values - current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append({"Name": metric_name, "Unit": metric.unit}) + if not config.disable_metric_extraction: + current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append({"Name": metric_name, "Unit": metric.unit}) + + num_metrics_in_current_body += 1 - should_serialize: bool = len(current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"]) == MAX_METRICS_PER_EVENT + should_serialize: bool = num_metrics_in_current_body == MAX_METRICS_PER_EVENT if should_serialize: event_batches.append(json.dumps(current_body)) current_body = create_body() + num_metrics_in_current_body = 0 - if not event_batches or current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"]: + if not event_batches or num_metrics_in_current_body > 0: event_batches.append(json.dumps(current_body)) return event_batches diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 532bdc4..d4c559d 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -23,6 +23,7 @@ def test_can_get_config_from_environment(monkeypatch): agent_endpoint = fake.word() ec2_metadata_endpoint = fake.word() namespace = fake.word() + disable_metric_extraction = True monkeypatch.setenv("AWS_EMF_ENABLE_DEBUG_LOGGING", str(debug_enabled)) monkeypatch.setenv("AWS_EMF_SERVICE_NAME", service_name) @@ -32,6 +33,7 @@ def test_can_get_config_from_environment(monkeypatch): monkeypatch.setenv("AWS_EMF_AGENT_ENDPOINT", agent_endpoint) monkeypatch.setenv("AWS_EMF_EC2_METADATA_ENDPOINT", ec2_metadata_endpoint) monkeypatch.setenv("AWS_EMF_NAMESPACE", namespace) + monkeypatch.setenv("AWS_EMF_DISABLE_METRIC_EXTRACTION", str(disable_metric_extraction)) # act result = get_config() @@ -45,6 +47,7 @@ def test_can_get_config_from_environment(monkeypatch): assert result.agent_endpoint == agent_endpoint assert result.ec2_metadata_endpoint == ec2_metadata_endpoint assert result.namespace == namespace + assert result.disable_metric_extraction == disable_metric_extraction def test_can_override_config(monkeypatch): @@ -57,6 +60,7 @@ def test_can_override_config(monkeypatch): monkeypatch.setenv("AWS_EMF_AGENT_ENDPOINT", fake.word()) monkeypatch.setenv("AWS_EMF_EC2_METADATA_ENDPOINT", fake.word()) monkeypatch.setenv("AWS_EMF_NAMESPACE", fake.word()) + monkeypatch.setenv("AWS_EMF_DISABLE_METRIC_EXTRACTION", str(True)) config = get_config() @@ -68,6 +72,7 @@ def test_can_override_config(monkeypatch): agent_endpoint = fake.word() ec2_metadata_endpoint = fake.word() namespace = fake.word() + disable_metric_extraction = False # act config.debug_logging_enabled = debug_enabled @@ -78,6 +83,7 @@ def test_can_override_config(monkeypatch): config.agent_endpoint = agent_endpoint config.ec2_metadata_endpoint = ec2_metadata_endpoint config.namespace = namespace + config.disable_metric_extraction = disable_metric_extraction # assert assert config.debug_logging_enabled == debug_enabled @@ -88,3 +94,4 @@ def test_can_override_config(monkeypatch): assert config.agent_endpoint == agent_endpoint assert config.ec2_metadata_endpoint == ec2_metadata_endpoint assert config.namespace == namespace + assert config.disable_metric_extraction == disable_metric_extraction diff --git a/tests/serializer/test_log_serializer.py b/tests/serializer/test_log_serializer.py index 2d7c05f..32795f7 100644 --- a/tests/serializer/test_log_serializer.py +++ b/tests/serializer/test_log_serializer.py @@ -1,3 +1,4 @@ +from aws_embedded_metrics.config import get_config from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.serializers.log_serializer import LogSerializer from faker import Faker @@ -177,6 +178,27 @@ def test_serialize_metrics_with_multiple_datapoints(): assert results == [json.dumps(expected)] +def test_serialize_metrics_with_aggregation_disabled(): + """Test log records don't contain metadata when aggregation is disabled.""" + # arrange + config = get_config() + config.disable_metric_extraction = True + + expected_key = fake.word() + expected_value = fake.random.randrange(0, 100) + + expected = {expected_key: expected_value} + + context = get_context() + context.put_metric(expected_key, expected_value) + + # act + result_json = serializer.serialize(context)[0] + + # assert + assert_json_equality(result_json, expected) + + # Test utility method diff --git a/tests/sinks/test_lambda_sink.py b/tests/sinks/test_lambda_sink.py index 2856c49..79aeb84 100644 --- a/tests/sinks/test_lambda_sink.py +++ b/tests/sinks/test_lambda_sink.py @@ -1,3 +1,6 @@ +from importlib import reload + +from aws_embedded_metrics import config from aws_embedded_metrics.sinks.lambda_sink import LambdaSink from aws_embedded_metrics.logger.metrics_context import MetricsContext from faker import Faker @@ -9,6 +12,8 @@ def test_accept_writes_to_stdout(capfd): # arrange + reload(config) + sink = LambdaSink() context = MetricsContext.empty() context.meta["Timestamp"] = 1 From 45859b02bb35e22ef01b51616375eb52d24f0b2b Mon Sep 17 00:00:00 2001 From: Daniel Roschka Date: Mon, 31 Aug 2020 08:26:23 +0200 Subject: [PATCH 2/2] Fix failing CI --- aws_embedded_metrics/serializers/log_serializer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aws_embedded_metrics/serializers/log_serializer.py b/aws_embedded_metrics/serializers/log_serializer.py index 6aa4176..fcc496a 100644 --- a/aws_embedded_metrics/serializers/log_serializer.py +++ b/aws_embedded_metrics/serializers/log_serializer.py @@ -19,7 +19,6 @@ from typing import Any, Dict, List - class LogSerializer(Serializer): @staticmethod def serialize(context: MetricsContext) -> List[str]: @@ -34,7 +33,7 @@ def serialize(context: MetricsContext) -> List[str]: dimensions_properties = {**dimensions_properties, **dimension_set} def create_body() -> Dict[str, Any]: - body = { + body: Dict[str, Any] = { **dimensions_properties, **context.properties, }