From 7f3f0ff32aa5ab8f6a00c2f30e8d241479da7b98 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 22 Jun 2022 16:25:59 +0200 Subject: [PATCH 1/9] docs(lint): initial markdownlint config --- .markdownlint.yaml | 244 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 .markdownlint.yaml diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 00000000000..3e99a5611f4 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,244 @@ +# Rules: https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md + +# Default state for all rules +default: true + +# Path to configuration file to extend +extends: null + +# MD001/heading-increment/header-increment - Heading levels should only increment by one level at a time +MD001: true + +# MD002/first-heading-h1/first-header-h1 - First heading should be a top-level heading +# NOTE: We use h2 due to font size +MD002: false + +# MD003/heading-style/header-style - Heading style +MD003: + # Heading style + style: "consistent" + +# MD004/ul-style - Unordered list style +MD004: + # List style + style: "consistent" + +# MD005/list-indent - Inconsistent indentation for list items at the same level +MD005: true + +# MD006/ul-start-left - Consider starting bulleted lists at the beginning of the line +MD006: true + +# MD007/ul-indent - Unordered list indentation +MD007: + # Spaces for indent + indent: 2 + # Whether to indent the first level of the list + start_indented: false + # Spaces for first level indent (when start_indented is set) + start_indent: 2 + +# MD009/no-trailing-spaces - Trailing spaces +MD009: + # Spaces for line break + br_spaces: 2 + # Allow spaces for empty lines in list items + list_item_empty_lines: false + # Include unnecessary breaks + strict: false + +# MD010/no-hard-tabs - Hard tabs +# NOTE: Mkdocs Material theme features like code annotations, tabbed content require it +MD010: false + +# MD011/no-reversed-links - Reversed link syntax +MD011: true + +# MD012/no-multiple-blanks - Multiple consecutive blank lines +MD012: + # Consecutive blank lines + maximum: 1 + +# MD013/line-length - Line length +MD013: + # Number of characters + line_length: 380 + # Number of characters for headings + heading_line_length: 80 + # Number of characters for code blocks + code_block_line_length: 180 + # Include code blocks + code_blocks: true + # Include tables + tables: false + # Include headings + headings: true + # Include headings + headers: true + # Strict length checking + strict: false + # Stern length checking + stern: false + +# MD014/commands-show-output - Dollar signs used before commands without showing output +MD014: true + +# MD018/no-missing-space-atx - No space after hash on atx style heading +MD018: true + +# MD019/no-multiple-space-atx - Multiple spaces after hash on atx style heading +MD019: true + +# MD020/no-missing-space-closed-atx - No space inside hashes on closed atx style heading +MD020: true + +# MD021/no-multiple-space-closed-atx - Multiple spaces inside hashes on closed atx style heading +MD021: true + +# MD022/blanks-around-headings/blanks-around-headers - Headings should be surrounded by blank lines +MD022: + # Blank lines above heading + lines_above: 1 + # Blank lines below heading + lines_below: 1 + +# MD023/heading-start-left/header-start-left - Headings must start at the beginning of the line +MD023: true + +# MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the same content +MD024: + # Only check sibling headings + allow_different_nesting: false + # Only check sibling headings + siblings_only: false + +# MD025/single-title/single-h1 - Multiple top-level headings in the same document +MD025: + # Heading level + level: 1 + # RegExp for matching title in front matter + front_matter_title: "^\\s*title\\s*[:=]" + +# MD026/no-trailing-punctuation - Trailing punctuation in heading +MD026: + # Punctuation characters + punctuation: ".,;:!。，；：！" + +# MD027/no-multiple-space-blockquote - Multiple spaces after blockquote symbol +MD027: true + +# MD028/no-blanks-blockquote - Blank line inside blockquote +MD028: true + +# MD029/ol-prefix - Ordered list item prefix +MD029: + # List style + style: "one_or_ordered" + +# MD030/list-marker-space - Spaces after list markers +MD030: + # Spaces for single-line unordered list items + ul_single: 1 + # Spaces for single-line ordered list items + ol_single: 1 + # Spaces for multi-line unordered list items + ul_multi: 1 + # Spaces for multi-line ordered list items + ol_multi: 1 + +# MD031/blanks-around-fences - Fenced code blocks should be surrounded by blank lines +MD031: + # Include list items + list_items: true + +# MD032/blanks-around-lists - Lists should be surrounded by blank lines +MD032: true + +# MD033/no-inline-html - Inline HTML +# NOTE: Some content like Logger '' triggers false positives +MD033: false + +# MD034/no-bare-urls - Bare URL used +MD034: true + +# MD035/hr-style - Horizontal rule style +MD035: + # Horizontal rule style + style: "consistent" + +# MD036/no-emphasis-as-heading/no-emphasis-as-header - Emphasis used instead of a heading +# NOTE: We use **** instead of yet another sub-heading that might not appear in the navigation. +# this is a trade-off we make to not a gigantic right-navigation +MD036: false + +# MD037/no-space-in-emphasis - Spaces inside emphasis markers +MD037: true + +# MD038/no-space-in-code - Spaces inside code span elements +# mkdocs-material requires these in tab content +MD038: false + +# MD039/no-space-in-links - Spaces inside link text +MD039: true + +# MD040/fenced-code-language - Fenced code blocks should have a language specified +MD040: true + +# MD041/first-line-heading/first-line-h1 - First line in a file should be a top-level heading +MD041: + # Heading level + level: 2 + # RegExp for matching title in front matter + front_matter_title: "^\\s*title\\s*[:=]" + +# MD042/no-empty-links - No empty links +# NOTE: Clipboard links like Lambda Layers use empty links +MD042: false + +# MD043/required-headings/required-headers - Required heading structure +# NOTE: Enforce our minimum headers across the docs +MD043: + # List of headings + headings: + [ + "*", + "## Key features", + "*", + "## Getting started", + "*", + "## Advanced", + "*", + "## Testing your code", + "*", + ] + +# MD044/proper-names - Proper names should have the correct capitalization +MD044: + # List of proper names + names: [] + # Include code blocks + code_blocks: true + # Include HTML elements + html_elements: true + +# MD045/no-alt-text - Images should have alternate text (alt text) +MD045: true + +# MD046/code-block-style - Code block style +# Material theme tabbed content feature use indented and simple use fenced; can't support both +MD046: false + +# MD047/single-trailing-newline - Files should end with a single newline character +MD047: true + +# MD048/code-fence-style - Code fence style +MD048: false + +# MD051/link-fragments - Link fragments should be valid +MD051: true + +# MD052/reference-links-images - Reference links and images should use a label that is defined +MD052: true + +# MD053/link-image-reference-definitions - Link and image reference definitions should be needed +MD053: true From 7f927aa53bc44799bb5b78e81d2e2f30c2e31933 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 22 Jun 2022 17:22:27 +0200 Subject: [PATCH 2/9] docs(lint): ignore minimum structure for typing doc --- docs/utilities/typing.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/utilities/typing.md b/docs/utilities/typing.md index c1b4dbad32b..a23d014afa6 100644 --- a/docs/utilities/typing.md +++ b/docs/utilities/typing.md @@ -3,6 +3,8 @@ title: Typing description: Utility --- + + This typing utility provides static typing classes that can be used to ease the development by providing the IDE type hints. ![Utilities Typing](../media/utilities_typing.png) From 0485d0653bb4e3bd923a0357db3567b96c212b96 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 22 Jun 2022 18:11:18 +0200 Subject: [PATCH 3/9] docs(lint): adjust markdownlint to our current conventions --- .markdownlint.yaml | 4 +-- docs/changelog.md | 2 ++ docs/core/event_handler/api_gateway.md | 44 +++++++++++++------------ docs/core/event_handler/appsync.md | 1 - docs/core/logger.md | 3 -- docs/core/metrics.md | 1 - docs/core/tracer.md | 2 -- docs/index.md | 6 +--- docs/roadmap.md | 12 ++++--- docs/tutorial/index.md | 24 +++++++------- docs/utilities/batch.md | 45 +++++++++----------------- docs/utilities/feature_flags.md | 5 --- docs/utilities/idempotency.md | 4 +-- docs/utilities/jmespath_functions.md | 1 + docs/utilities/parameters.md | 7 +--- docs/utilities/parser.md | 38 +++++++++++----------- docs/utilities/validation.md | 24 +++++++------- 17 files changed, 101 insertions(+), 122 deletions(-) diff --git a/.markdownlint.yaml b/.markdownlint.yaml index 3e99a5611f4..4d571206e07 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -32,7 +32,7 @@ MD006: true # MD007/ul-indent - Unordered list indentation MD007: # Spaces for indent - indent: 2 + indent: 4 # Whether to indent the first level of the list start_indented: false # Spaces for first level indent (when start_indented is set) @@ -66,7 +66,7 @@ MD013: # Number of characters for headings heading_line_length: 80 # Number of characters for code blocks - code_block_line_length: 180 + code_block_line_length: 265 # Include code blocks code_blocks: true # Include tables diff --git a/docs/changelog.md b/docs/changelog.md index c2705ba58cb..e313bea09a9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,2 +1,4 @@ [comment]: <> (Includes Changelog content entire file as a snippet) + + --8<-- "CHANGELOG.md" diff --git a/docs/core/event_handler/api_gateway.md b/docs/core/event_handler/api_gateway.md index 4f86dc8fdf3..0ae3647419c 100644 --- a/docs/core/event_handler/api_gateway.md +++ b/docs/core/event_handler/api_gateway.md @@ -18,10 +18,12 @@ Event handler for Amazon API Gateway REST and HTTP APIs, and Application Loader ### Required resources -You must have an existing [API Gateway Proxy integration](https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html){target="_blank"} or [ALB](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/lambda-functions.html){target="_blank"} configured to invoke your Lambda function. There is no additional permissions or dependencies required to use this utility. +You must have an existing [API Gateway Proxy integration](https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html){target="_blank"} or [ALB](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/lambda-functions.html){target="_blank"} configured to invoke your Lambda function. This is the sample infrastructure for API Gateway we are using for the examples in this documentation. +???+ info "There is no additional permissions or dependencies required to use this utility." + ```yaml title="AWS Serverless Application Model (SAM) example" AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 @@ -35,7 +37,7 @@ Globals: AllowHeaders: "'Content-Type,Authorization,X-Amz-Date'" MaxAge: "'300'" BinaryMediaTypes: # see Binary responses section - - '*~1*' # converts to */* for any binary type + - '*~1*' # converts to */* for any binary type Function: Timeout: 5 Runtime: python3.8 @@ -59,8 +61,11 @@ Resources: ApiEvent: Type: Api Properties: - Path: /{proxy+} # Send requests on any path to the lambda function - Method: ANY # Send requests using any http method to the lambda function + # NOTE: this is a catch-all rule to simply the documentation. + # explicit routes and methods are recommended for prod instead + # for example, Path: /hello, Method: GET + Path: /{proxy+} # Send requests on any path to the lambda function + Method: ANY # Send requests using any http method to the lambda function ``` ### Event Resolvers @@ -355,7 +360,9 @@ You can also combine nested paths with greedy regex to catch in between routes. ... } ``` + ### HTTP Methods + You can use named decorators to specify the HTTP method that should be handled in your functions. As well as the `get` method already shown above, you can use `post`, `put`, `patch`, `delete`, and `patch`. @@ -487,7 +494,6 @@ def lambda_handler(event, context): return app.resolve(event, context) ``` - ### Handling not found routes By default, we return `404` for any unmatched route. @@ -528,7 +534,6 @@ def lambda_handler(event, context): return app.resolve(event, context) ``` - ### Exception handling You can use **`exception_handler`** decorator with any Python exception. This allows you to handle a common exception outside your route, for example validation errors. @@ -754,13 +759,13 @@ For convenience, these are the default values when using `CORSConfig` to enable ???+ warning Always configure `allow_origin` when using in production. -Key | Value | Note -------------------------------------------------- | --------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- -**[allow_origin](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin){target="_blank"}**: `str` | `*` | Only use the default value for development. **Never use `*` for production** unless your use case requires it -**[allow_headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Headers){target="_blank"}**: `List[str]` | `[Authorization, Content-Type, X-Amz-Date, X-Api-Key, X-Amz-Security-Token]` | Additional headers will be appended to the default list for your convenience -**[expose_headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Expose-Headers){target="_blank"}**: `List[str]` | `[]` | Any additional header beyond the [safe listed by CORS specification](https://developer.mozilla.org/en-US/docs/Glossary/CORS-safelisted_response_header){target="_blank"}. -**[max_age](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Max-Age){target="_blank"}**: `int` | `` | Only for pre-flight requests if you choose to have your function to handle it instead of API Gateway -**[allow_credentials](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials){target="_blank"}**: `bool` | `False` | Only necessary when you need to expose cookies, authorization headers or TLS client certificates. +| Key | Value | Note | +| -------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **[allow_origin](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin){target="_blank"}**: `str` | `*` | Only use the default value for development. **Never use `*` for production** unless your use case requires it | +| **[allow_headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Headers){target="_blank"}**: `List[str]` | `[Authorization, Content-Type, X-Amz-Date, X-Api-Key, X-Amz-Security-Token]` | Additional headers will be appended to the default list for your convenience | +| **[expose_headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Expose-Headers){target="_blank"}**: `List[str]` | `[]` | Any additional header beyond the [safe listed by CORS specification](https://developer.mozilla.org/en-US/docs/Glossary/CORS-safelisted_response_header){target="_blank"}. | +| **[max_age](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Max-Age){target="_blank"}**: `int` | `` | Only for pre-flight requests if you choose to have your function to handle it instead of API Gateway | +| **[allow_credentials](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials){target="_blank"}**: `bool` | `False` | Only necessary when you need to expose cookies, authorization headers or TLS client certificates. | ### Fine grained responses @@ -1132,7 +1137,6 @@ This sample project contains a Users function with two distinct set of routes, ` === "Project layout" - ```python hl_lines="1 8 10 12-15" . ├── Pipfile # project app & dev dependencies; poetry, pipenv, etc. @@ -1308,7 +1312,7 @@ _**Downsides**_ * **Cold starts**. Frequent deployments and/or high load can diminish the benefit of monolithic functions depending on your latency requirements, due to [Lambda scaling model](https://docs.aws.amazon.com/lambda/latest/dg/invocation-scaling.html){target="_blank"}. Always load test to pragmatically balance between your customer experience and development cognitive load. * **Granular security permissions**. The micro function approach enables you to use fine-grained permissions & access controls, separate external dependencies & code signing at the function level. Conversely, you could have multiple functions while duplicating the final code artifact in a monolithic approach. - - Regardless, least privilege can be applied to either approaches. + * Regardless, least privilege can be applied to either approaches. * **Higher risk per deployment**. A misconfiguration or invalid import can cause disruption if not caught earlier in automated testing. Multiple functions can mitigate misconfigurations but they would still share the same code artifact. You can further minimize risks with multiple environments in your CI/CD pipeline. #### Micro function @@ -1317,20 +1321,20 @@ _**Downsides**_ A micro function means that your final code artifact will be different to each function deployed. This is generally the approach to start if you're looking for fine-grain control and/or high load on certain parts of your service. -_**Benefits**_ +**Benefits** * **Granular scaling**. A micro function can benefit from the [Lambda scaling model](https://docs.aws.amazon.com/lambda/latest/dg/invocation-scaling.html){target="_blank"} to scale differently depending on each part of your application. Concurrency controls and provisioned concurrency can also be used at a granular level for capacity management. * **Discoverability**. Micro functions are easier do visualize when using distributed tracing. Their high-level architectures can be self-explanatory, and complexity is highly visible — assuming each function is named to the business purpose it serves. * **Package size**. An independent function can be significant smaller (KB vs MB) depending on external dependencies it require to perform its purpose. Conversely, a monolithic approach can benefit from [Lambda Layers](https://docs.aws.amazon.com/lambda/latest/dg/invocation-layers.html){target="_blank"} to optimize builds for external dependencies. -_**Downsides**_ +**Downsides** -* **Upfront investment**. Python ecosystem doesn't use a bundler — you need a custom build tooling to ensure each function only has what it needs and account for [C bindings for runtime compatibility](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html){target="_blank"}. Operations become more elaborate — you need to standardize tracing labels/annotations, structured logging, and metrics to pinpoint root causes. - - Engineering discipline is necessary for both approaches. Micro-function approach however requires further attention in consistency as the number of functions grow, just like any distributed system. +* **Upfront investment**. You need custom build tooling to bundle assets, including [C bindings for runtime compatibility](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html){target="_blank"}. `Operations become more elaborate — you need to standardize tracing labels/annotations, structured logging, and metrics to pinpoint root causes. + * Engineering discipline is necessary for both approaches. Micro-function approach however requires further attention in consistency as the number of functions grow, just like any distributed system. * **Harder to share code**. Shared code must be carefully evaluated to avoid unnecessary deployments when that changes. Equally, if shared code isn't a library, your development, building, deployment tooling need to accommodate the distinct layout. * **Slower safe deployments**. Safely deploying multiple functions require coordination — AWS CodeDeploy deploys and verifies each function sequentially. This increases lead time substantially (minutes to hours) depending on the deployment strategy you choose. You can mitigate it by selectively enabling it in prod-like environments only, and where the risk profile is applicable. - - Automated testing, operational and security reviews are essential to stability in either approaches. + * Automated testing, operational and security reviews are essential to stability in either approaches. ## Testing your code diff --git a/docs/core/event_handler/appsync.md b/docs/core/event_handler/appsync.md index 95457aa7736..f3203e37834 100644 --- a/docs/core/event_handler/appsync.md +++ b/docs/core/event_handler/appsync.md @@ -770,7 +770,6 @@ Let's assume you have `app.py` as your Lambda function entrypoint and routes in app.resolve(event, context) ``` - ## Testing your code You can test your resolvers by passing a mocked or actual AppSync Lambda event that you're expecting. diff --git a/docs/core/logger.md b/docs/core/logger.md index 0edc4aa3ba7..bc42225edcf 100644 --- a/docs/core/logger.md +++ b/docs/core/logger.md @@ -736,7 +736,6 @@ You might want to continue to use the same date formatting style, or override `l Logger allows you to either change the format or suppress the following keys altogether at the initialization: `location`, `timestamp`, `level`, `xray_trace_id`. - === "lambda_handler.py" ```python hl_lines="7 10" from aws_lambda_powertools import Logger @@ -902,7 +901,6 @@ For exceptional cases where you want to completely replace our formatter logic, ???+ warning You will need to implement `append_keys`, `clear_state`, override `format`, and optionally `remove_keys` to keep the same feature set Powertools Logger provides. This also means keeping state of logging keys added. - === "collect.py" ```python hl_lines="5 7 9-10 13 17 21 24 35" @@ -1084,7 +1082,6 @@ def handler(event: Dict, context: LambdaContext) -> List: You can copy the Logger setup to all or sub-sets of registered external loggers. Use the `copy_config_to_registered_logger` method to do this. By default all registered loggers will be modified. You can change this behaviour by providing `include` and `exclude` attributes. You can also provide optional `log_level` attribute external loggers will be configured with. - ```python hl_lines="10" title="Cloning Logger config to all other registered standard loggers" import logging diff --git a/docs/core/metrics.md b/docs/core/metrics.md index 713ae874fe6..63e0c02aba9 100644 --- a/docs/core/metrics.md +++ b/docs/core/metrics.md @@ -66,7 +66,6 @@ Metric has two global settings that will be used across all metrics emitted: metrics = Metrics(namespace="ServerlessAirline", service="orders") # Sets metric namespace, and service as a metric dimension ``` - ### Creating metrics You can create metrics using `add_metric`, and you can create dimensions for all your aggregate metrics using `add_dimension` method. diff --git a/docs/core/tracer.md b/docs/core/tracer.md index 363611bbbc0..a773ecb52a8 100644 --- a/docs/core/tracer.md +++ b/docs/core/tracer.md @@ -101,7 +101,6 @@ def collect_payment(charge_id): The serialization is performed by aws-xray-sdk via `jsonpickle` module. This can cause side effects for file-like objects like boto S3 `StreamingBody`, where its response will be read only once during serialization. - ### Asynchronous and generator functions ???+ warning @@ -246,7 +245,6 @@ def handler(event, context): ec2_api_calls() ``` - ### Tracing aiohttp requests ???+ info diff --git a/docs/index.md b/docs/index.md index e2b82b35b0d..60352f9d818 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,6 @@ A suite of utilities for AWS Lambda functions to ease adopting best practices su Check out [this detailed blog post](https://aws.amazon.com/blogs/opensource/simplifying-serverless-best-practices-with-lambda-powertools/) with a practical example. - ## Install Powertools is available in the following formats: @@ -22,7 +21,6 @@ Powertools is available in the following formats: When using Layers, you can add Lambda Powertools as a dev dependency (or as part of your virtual env) to not impact the development process. - ### Lambda Layer [Lambda Layer](https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html){target="_blank"} is a .zip file archive that can contain additional code, pre-packaged dependencies, data, or configuration files. Layers promote code sharing and separation of responsibilities so that you can iterate faster on writing business logic. @@ -187,7 +185,6 @@ You can include Lambda Powertools Lambda Layer using [AWS Lambda Console](https: Lambda Powertools Lambda Layer do not include `pydantic` library - required dependency for the `parser` utility. See [SAR](#sar) option instead. - #### SAR Serverless Application Repository (SAR) App deploys a CloudFormation stack with a copy of our Lambda Layer in your AWS account and region. @@ -205,7 +202,6 @@ Despite having more steps compared to the [public Layer ARN](#lambda-layer) opti ???+ tip You can create a shared Lambda Layers stack and make this along with other account level layers stack. - If using SAM, you can include this SAR App as part of your shared Layers stack, and lock to a specific semantic version. Once deployed, it'll be available across the account this is deployed to. === "SAM" @@ -460,7 +456,7 @@ from aws_lambda_powertools.logging.logger import set_package_logger set_package_logger() # (1) ``` -1. :information_source: this will configure our `aws_lambda_powertools` logger with debug. +1. :information_source: this will configure our `aws_lambda_powertools` logger with debug. ## Tenets diff --git a/docs/roadmap.md b/docs/roadmap.md index 231a63927ec..bc0cfb55be2 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -4,7 +4,6 @@ This is our public roadmap that outlines the high level direction we are working towards, namely [Themes](#themes). We update this document when our priorities change: security and stability is our top priority. - [See our latest list of activities »](https://github.com/orgs/awslabs/projects/51/views/1?query=is%3Aopen+sort%3Aupdated-desc){target="_blank"} ## Themes @@ -25,7 +24,9 @@ We will remove support for Python 3.6 after July 18th, following AWS Lambda [dep ### Reduce release operational overhead -We are working on a consistent label and automation strategy across all Lambda Powertools projects ([Java](https://awslabs.github.io/aws-lambda-powertools-java/){target="_blank"}, [TypeScript](https://awslabs.github.io/aws-lambda-powertools-typescript/latest/){target="_blank"}). This will be our baseline to automate areas where we don't need human intervention, and reduce our manual effort to areas where clear communication is crucial. +We are working on a consistent label and automation strategy across all Lambda Powertools projects ([Java](https://awslabs.github.io/aws-lambda-powertools-java/){target="_blank"}, [TypeScript](https://awslabs.github.io/aws-lambda-powertools-typescript/latest/){target="_blank"}). + +This will be our baseline to automate areas where we don't need human intervention, and reduce our manual effort to areas where clear communication is crucial. ### Revamp roadmap @@ -71,8 +72,10 @@ graph LR Our end-to-end mechanism follows four major steps: -* **Feature Request**. Ideas start with a [feature request issue template](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=feature-request%2Ctriage&template=feature_request.yml&title=Feature+request%3A+TITLE){target="_blank"} to highlight their use case at a high level. Maintainers review each request based on **(1)** [project tenets](index.md#tenets){target="_blank"}, **(2)** customers reaction (👍) and use cases, and comment whether we'll need a RFC for further discussion before any work begins. -* **Request-for-comments (RFC)**. Design proposals use our [RFC issue template](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=RFC%2Ctriage&template=rfc.yml&title=RFC%3A+TITLE){target="_blank"} to describe its implementation, challenges, developer experience, dependencies, and alternative solutions. This helps refine the initial idea with community feedback before a decision is made. +* **Feature Request**. Ideas start with a [feature request](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=feature-request%2Ctriage&template=feature_request.yml&title=Feature+request%3A+TITLE){target="_blank"} to outline their use case at a high level. For complex use cases, maintainers might ask for/write a RFC. + * Maintainers review requests based on [project tenets](index.md#tenets){target="_blank"}, customers reaction (👍), and use cases. +* **Request-for-comments (RFC)**. Design proposals use our [RFC issue template](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=RFC%2Ctriage&template=rfc.yml&title=RFC%3A+TITLE){target="_blank"} to describe its implementation, challenges, developer experience, dependencies, and alternative solutions. + * This helps refine the initial idea with community feedback before a decision is made. * **Decision**. After carefully reviewing and discussing them, maintainers make a final decision on whether to start implementation, defer or reject it, and update everyone with the next steps. * **Implementation**. For approved features, maintainers give priority to the original authors for implementation unless it is a sensitive task that is best handled by maintainers. @@ -84,7 +87,6 @@ The AWS Lambda Powertools team values feedback and guidance from its community o We determine the high-level direction for our open roadmap based on customer feedback and popularity (👍🏽 and comments), security and operational impacts, and business value. Where features don’t meet our goals and longer-term strategy, we will communicate that clearly and openly as quickly as possible with an explanation of why the decision was made. - ## FAQs **Q: Why did you build this?** diff --git a/docs/tutorial/index.md b/docs/tutorial/index.md index 5ea8ec7f2fa..e6f7cbfed29 100644 --- a/docs/tutorial/index.md +++ b/docs/tutorial/index.md @@ -82,6 +82,7 @@ When API Gateway receives a HTTP GET request on `/hello` route, Lambda will call ???+ warning For simplicity, we do not set up authentication and authorization! You can find more information on how to implement it on [AWS SAM documentation](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-controlling-access-to-apis.html){target="_blank"}. + ### Run your code At each point, you have two ways to run your code: locally and within your AWS account. @@ -106,7 +107,6 @@ As a result, a local API endpoint will be exposed and you can invoke it using yo ???+ info To learn more about local testing, please visit the [AWS SAM CLI local testing](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-cli-command-reference-sam-local-start-api.html) documentation. - #### Live test First, you need to deploy your application into your AWS Account by issuing `sam build && sam deploy --guided` command. This command builds a ZIP package of your source code, and deploy it to your AWS Account. @@ -357,9 +357,9 @@ Lastly, we used `return app.resolve(event, context)` so Event Handler can resolv From here, we could handle [404 routes](../core/event_handler/api_gateway.md#handling-not-found-routes){target="_blank"}, [error handling](../core/event_handler/api_gateway.md#exception-handling){target="_blank"}, [access query strings, payload](../core/event_handler/api_gateway.md#accessing-request-details){target="_blank"}, etc. - ???+ tip If you'd like to learn how python decorators work under the hood, you can follow [Real Python](https://realpython.com/primer-on-python-decorators/)'s article. + ## Structured Logging Over time, you realize that searching logs as text results in poor observability, it's hard to create metrics from, enumerate common exceptions, etc. @@ -443,7 +443,6 @@ So far, so good! We can take a step further now by adding additional context to We could start by creating a dictionary with Lambda context information or something from the incoming event, which should always be logged. Additional attributes could be added on every `logger.info` using `extra` keyword like in any standard Python logger. - ### Simplifying with Logger ???+ question "Surely this could be easier, right?" @@ -485,7 +484,6 @@ Let's break this down: * **L22**: We also instruct Logger to use the incoming API Gateway Request ID as a [correlation id](../core/logger.md##set_correlation_id-method) automatically. * **L22**: Since we're in dev, we also use `log_event=True` to automatically log each incoming request for debugging. This can be also set via [environment variables](./index.md#environment-variables){target="_blank"}. - This is how the logs would look like now: ```json title="Our logs are now structured consistently" @@ -707,7 +705,9 @@ Let's break it down: ???+ info If you want to understand how the Lambda execution environment (sandbox) works and why cold starts can occur, see this [blog series on Lambda performance](https://aws.amazon.com/blogs/compute/operating-lambda-performance-optimization-part-1/). -Repeat the process of building, deploying, and invoking your application via the API endpoint. Within the [AWS X-Ray Console](https://console.aws.amazon.com/xray/home#/traces/){target="_blank"}, you should now be able to group traces by the `User` and `ColdStart` annotation. +Repeat the process of building, deploying, and invoking your application via the API endpoint. + +Within the [AWS X-Ray Console](https://console.aws.amazon.com/xray/home#/traces/){target="_blank"}, you should now be able to group traces by the `User` and `ColdStart` annotation. ![Filtering traces by annotations](../media/tracer_xray_sdk_enriched.png) @@ -772,7 +772,6 @@ Lambda Powertools optimizes for Lambda compute environment. As such, we add thes Repeat the process of building, deploying, and invoking your application via the API endpoint. Within the [AWS X-Ray Console](https://console.aws.amazon.com/xray/home#/traces/){target="_blank"}, you should see a similar view: - ![AWS X-Ray Console trace view using Lambda Powertools Tracer](../media/tracer_utility_showcase_2.png) ???+ tip @@ -791,7 +790,7 @@ From here, you can browse to specific logs in CloudWatch Logs Insight, Metrics D Let's add custom metrics to better understand our application and business behavior (e.g. number of reservations, etc.). -Out of the box, AWS Lambda adds [invocation, performance, and concurrency metrics](https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics.html#monitoring-metrics-types){target="_blank"}. Amazon API Gateway also adds [general metrics at the aggregate level](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-metrics-and-dimensions.html#api-gateway-metrics) such as latency, number of requests received, etc. +By default, AWS Lambda adds [invocation and performance metrics](https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics.html#monitoring-metrics-types){target="_blank"}, and Amazon API Gateway adds [latency and some HTTP metrics](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-metrics-and-dimensions.html#api-gateway-metrics). ???+ tip You can [optionally enable detailed metrics](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-metrics-and-dimensions.html#api-gateway-metricdimensions){target="_blank"} per each API route, stage, and method in API Gateway. @@ -915,7 +914,8 @@ There's a lot going on, let's break this down: * **L10**: We define a container where all of our application metrics will live `MyApp`, a.k.a [Metrics Namespace](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html){target="_blank"}. * **L14**: We initialize a CloudWatch client to send metrics later. -* **L19-47**: We create a custom function to prepare and send `ColdStart` and `SuccessfulGreetings` metrics using CloudWatch expected data structure. We also set [dimensions](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#Dimension){target="_blank"} of these metrics - Think of them as metadata to define to slice and dice them later; an unique metric is a combination of metric name + metric dimension(s). +* **L19-47**: We create a custom function to prepare and send `ColdStart` and `SuccessfulGreetings` metrics using CloudWatch expected data structure. We also set [dimensions](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#Dimension){target="_blank"} of these metrics. + * Think of them as metadata to define to slice and dice them later; an unique metric is a combination of metric name + metric dimension(s). * **L55,64**: We call our custom function to create metrics for every greeting received. ???+ question @@ -988,7 +988,9 @@ That's a lot less boilerplate code! Let's break this down: * **L33**: We use `@metrics.log_metrics` decorator to ensure that our metrics are aligned with the EMF output and validated before-hand, like in case we forget to set namespace, or accidentally use a metric unit as a string that doesn't exist in CloudWatch. * **L33**: We also use `capture_cold_start_metric=True` so we don't have to handle that logic either. Note that [Metrics](../core/metrics.md){target="_blank"} does not publish a warm invocation metric (ColdStart=0) for cost reasons. As such, treat the absence (sparse metric) as a non-cold start invocation. -Repeat the process of building, deploying, and invoking your application via the API endpoint a few times to generate metrics - [Artillery](https://www.artillery.io/){target="_blank"} and [K6.io](https://k6.io/open-source){target="_blank"} are quick ways to generate some load. Within [CloudWatch Metrics view](https://console.aws.amazon.com/cloudwatch/home#metricsV2:graph=~()){target="_blank}, you should see `MyApp` custom namespace with your custom metrics there and `SuccessfulGreetings` available to graph. +Repeat the process of building, deploying, and invoking your application via the API endpoint a few times to generate metrics - [Artillery](https://www.artillery.io/){target="_blank"} and [K6.io](https://k6.io/open-source){target="_blank"} are quick ways to generate some load. + +Within [CloudWatch Metrics view](https://console.aws.amazon.com/cloudwatch/home#metricsV2:graph=~()){target="_blank}, you should see `MyApp` custom namespace with your custom metrics there and `SuccessfulGreetings` available to graph. ![Custom Metrics Example](../media/metrics_utility_showcase.png) @@ -1024,7 +1026,7 @@ If you're curious about how the EMF portion of your function logs look like, you } ``` -# Final considerations +## Final considerations We covered a lot of ground here and we only scratched the surface of the feature set available within Lambda Powertools. @@ -1038,4 +1040,4 @@ This requires a change in mindset to ensure operational excellence is part of th Lambda Powertools is largely designed to make some of these practices easier to adopt from day 1. ???+ question "Have ideas for other tutorials?" - You can open up a [documentation issue](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=documentation&template=documentation-improvements.md&title=Tutorial%20Suggestion){target="_blank"}, or connect with us on the [AWS Developers Slack](https://github.com/awslabs/aws-lambda-powertools-python/#connect) at `lambda-powertools` channel, or via e-mail [aws-lambda-powertools-feedback@amazon.com](mailto:aws-lambda-powertools-feedback@amazon.com). + You can open up a [documentation issue](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=documentation&template=documentation-improvements.md&title=Tutorial%20Suggestion){target="_blank"}, or via e-mail [aws-lambda-powertools-feedback@amazon.com](mailto:aws-lambda-powertools-feedback@amazon.com). diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 14dc80bdb11..ce2e76e25d4 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -20,9 +20,11 @@ If your function fails to process any message from the batch, the entire batch r With this utility, batch records are processed individually – only messages that failed to be processed return to the queue or stream for a further retry. This works when two mechanisms are in place: -1. `ReportBatchItemFailures` is set in your SQS, Kinesis, or DynamoDB event source properties -2. [A specific response](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#sqs-batchfailurereporting-syntax){target="_blank"} is returned so Lambda knows which records should not be deleted during partial responses +1. `ReportBatchItemFailures` is set in your SQS, Kinesis, or DynamoDB event source properties +2. [A specific response](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#sqs-batchfailurereporting-syntax){target="_blank"} is returned so Lambda knows which records should not be deleted during partial responses + + ???+ warning "Warning: This utility lowers the chance of processing records more than once; it does not guarantee it" We recommend implementing processing logic in an [idempotent manner](idempotency.md){target="_blank"} wherever possible. @@ -38,7 +40,6 @@ You do not need any additional IAM permissions to use this utility, except for w The remaining sections of the documentation will rely on these samples. For completeness, this demonstrates IAM permissions and Dead Letter Queue where batch records will be sent after 2 retries were attempted. - === "SQS" ```yaml title="template.yaml" hl_lines="31-32" @@ -220,10 +221,10 @@ The remaining sections of the documentation will rely on these samples. For comp Processing batches from SQS works in four stages: -1. Instantiate **`BatchProcessor`** and choose **`EventType.SQS`** for the event type -2. Define your function to handle each batch record, and use [`SQSRecord`](data_classes.md#sqs){target="_blank"} type annotation for autocompletion -3. Use either **`batch_processor`** decorator or your instantiated processor as a context manager to kick off processing -4. Return the appropriate response contract to Lambda via **`.response()`** processor method +1. Instantiate **`BatchProcessor`** and choose **`EventType.SQS`** for the event type +2. Define your function to handle each batch record, and use [`SQSRecord`](data_classes.md#sqs){target="_blank"} type annotation for autocompletion +3. Use either **`batch_processor`** decorator or your instantiated processor as a context manager to kick off processing +4. Return the appropriate response contract to Lambda via **`.response()`** processor method ???+ info This code example optionally uses Tracer and Logger for completion. @@ -350,10 +351,10 @@ Processing batches from SQS works in four stages: Processing batches from Kinesis works in four stages: -1. Instantiate **`BatchProcessor`** and choose **`EventType.KinesisDataStreams`** for the event type -2. Define your function to handle each batch record, and use [`KinesisStreamRecord`](data_classes.md#kinesis-streams){target="_blank"} type annotation for autocompletion -3. Use either **`batch_processor`** decorator or your instantiated processor as a context manager to kick off processing -4. Return the appropriate response contract to Lambda via **`.response()`** processor method +1. Instantiate **`BatchProcessor`** and choose **`EventType.KinesisDataStreams`** for the event type +2. Define your function to handle each batch record, and use [`KinesisStreamRecord`](data_classes.md#kinesis-streams){target="_blank"} type annotation for autocompletion +3. Use either **`batch_processor`** decorator or your instantiated processor as a context manager to kick off processing +4. Return the appropriate response contract to Lambda via **`.response()`** processor method ???+ info This code example optionally uses Tracer and Logger for completion. @@ -433,7 +434,6 @@ Processing batches from Kinesis works in four stages: } ``` - === "Sample event" ```json @@ -475,15 +475,14 @@ Processing batches from Kinesis works in four stages: } ``` - ### Processing messages from DynamoDB Processing batches from Kinesis works in four stages: -1. Instantiate **`BatchProcessor`** and choose **`EventType.DynamoDBStreams`** for the event type -2. Define your function to handle each batch record, and use [`DynamoDBRecord`](data_classes.md#dynamodb-streams){target="_blank"} type annotation for autocompletion -3. Use either **`batch_processor`** decorator or your instantiated processor as a context manager to kick off processing -4. Return the appropriate response contract to Lambda via **`.response()`** processor method +1. Instantiate **`BatchProcessor`** and choose **`EventType.DynamoDBStreams`** for the event type +2. Define your function to handle each batch record, and use [`DynamoDBRecord`](data_classes.md#dynamodb-streams){target="_blank"} type annotation for autocompletion +3. Use either **`batch_processor`** decorator or your instantiated processor as a context manager to kick off processing +4. Return the appropriate response contract to Lambda via **`.response()`** processor method ???+ info This code example optionally uses Tracer and Logger for completion. @@ -569,7 +568,6 @@ Processing batches from Kinesis works in four stages: } ``` - === "Sample event" ```json @@ -638,7 +636,6 @@ All records in the batch will be passed to this handler for processing, even if All processing logic will and should be performed by the `record_handler` function. - ## Advanced ### Pydantic integration @@ -647,7 +644,6 @@ You can bring your own Pydantic models via **`model`** parameter when inheriting Inheritance is importance because we need to access message IDs and sequence numbers from these records in the event of failure. Mypy is fully integrated with this utility, so it should identify whether you're passing the incorrect Model. - === "SQS" ```python hl_lines="5 9-10 12-19 21 27" @@ -789,7 +785,6 @@ Use the context manager to access a list of all returned values from your `recor * **When successful**. We will include a tuple with `success`, the result of `record_handler`, and the batch record * **When failed**. We will include a tuple with `fail`, exception as a string, and the batch record - ```python hl_lines="31-38" title="Accessing processed messages via context manager" import json @@ -833,7 +828,6 @@ def lambda_handler(event, context: LambdaContext): return processor.response() ``` - ### Extending BatchProcessor You might want to bring custom logic to the existing `BatchProcessor` to slightly override how we handle successes and failures. @@ -958,7 +952,6 @@ When using Tracer to capture responses for each batch record processing, you mig If that's the case, you can configure [Tracer to disable response auto-capturing](../core/tracer.md#disabling-response-auto-capture){target="_blank"}. - ```python hl_lines="14" title="Disabling Tracer response auto-capturing" import json @@ -1123,8 +1116,6 @@ Given a SQS batch where the first batch record succeeds and the second fails pro } ``` - - ## FAQ ### Choosing between decorator and context manager @@ -1150,13 +1141,11 @@ class MyProcessor(BatchProcessor): return super().failure_handler(record, exception) ``` - ## Legacy ???+ tip This is kept for historical purposes. Use the new [BatchProcessor](#processing-messages-from-sqs) instead. - ### Migration guide ???+ info @@ -1175,7 +1164,6 @@ You can migrate in three steps: 2. If you were using **`PartialSQSProcessor`** you can now use **`BatchProcessor`** 3. Change your Lambda Handler to return the new response format - === "Decorator: Before" ```python hl_lines="1 6" @@ -1207,7 +1195,6 @@ You can migrate in three steps: return processor.response() ``` - === "Context manager: Before" ```python hl_lines="1-2 4 14 19" diff --git a/docs/utilities/feature_flags.md b/docs/utilities/feature_flags.md index 95efc5d051c..1d586d9377d 100644 --- a/docs/utilities/feature_flags.md +++ b/docs/utilities/feature_flags.md @@ -387,7 +387,6 @@ You can use `get_enabled_features` method for scenarios where you need a list of Feature flags can return any JSON values when `boolean_type` parameter is set to `false`. These can be dictionaries, list, string, integers, etc. - === "app.py" ```python hl_lines="3 9 13 16 18" @@ -593,7 +592,6 @@ Action | Equivalent expression **VALUE_IN_KEY** | `lambda a, b: b in a` **VALUE_NOT_IN_KEY** | `lambda a, b: b not in a` - ???+ info The `**key**` and `**value**` will be compared to the input from the `**context**` parameter. @@ -655,7 +653,6 @@ For this to work, you need to use a JMESPath expression via the `envelope` param } ``` - ### Built-in store provider ???+ info @@ -678,7 +675,6 @@ Parameter | Default | Description **jmespath_options** | `None` | For advanced use cases when you want to bring your own [JMESPath functions](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} **logger** | `logging.Logger` | Logger to use for debug. You can optionally supply an instance of Powertools Logger. - ```python hl_lines="21-27" title="AppConfigStore sample" from botocore.config import Config @@ -778,7 +774,6 @@ Method | When to use | Requires new deployment on changes | Supported services **[Parameters utility](parameters.md)** | Access to secrets, or fetch parameters in different formats from AWS System Manager Parameter Store or Amazon DynamoDB. | No | Parameter Store, DynamoDB, Secrets Manager, AppConfig **Feature flags utility** | Rule engine to define when one or multiple features should be enabled depending on the input. | No | AppConfig - ## Deprecation list when GA Breaking change | Recommendation diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 4b03b66abd4..a5ed14b9150 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -350,7 +350,6 @@ Imagine the function executes successfully, but the client never receives the re } ``` - ### Idempotency request flow This sequence diagram shows an example flow of what happens in the payment scenario: @@ -367,7 +366,6 @@ The client was successful in receiving the result after the retry. Since the Lam If you are using the `idempotent` decorator on your Lambda handler, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. This means that new invocations will execute your code again despite having the same payload. If you don't want the record to be deleted, you need to catch exceptions within the idempotent function and return a successful response. - ![Idempotent sequence exception](../media/idempotent_sequence_exception.png) If you are using `idempotent_function`, any unhandled exceptions that are raised _inside_ the decorated function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if retried. @@ -886,12 +884,12 @@ def lambda_handler(event, context): ???+ tip "Tip: JMESPath Powertools functions are also available" Built-in functions known in the validation utility like `powertools_json`, `powertools_base64`, `powertools_base64_gzip` are also available to use in this utility. - ## Testing your code The idempotency utility provides several routes to test your code. ### Disabling the idempotency utility + When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable `POWERTOOLS_IDEMPOTENCY_DISABLED` with a truthy value. If you prefer setting this for specific tests, and are using Pytest, you can use [monkeypatch](https://docs.pytest.org/en/latest/monkeypatch.html) fixture: diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 03b5fce1fd5..eee88c13cfb 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -107,6 +107,7 @@ Envelope | JMESPath expression ## Advanced ### Built-in JMESPath functions + You can use our built-in JMESPath functions within your expressions to do exactly that to decode JSON Strings, base64, and uncompress gzip data. ???+ info diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 36990fdd2cb..2559044b632 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -3,7 +3,7 @@ title: Parameters description: Utility --- - + The parameters utility provides high-level functions to retrieve one or multiple parameter values from [AWS Systems Manager Parameter Store](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html){target="_blank"}, [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/){target="_blank"}, [AWS AppConfig](https://docs.aws.amazon.com/appconfig/latest/userguide/what-is-appconfig.html){target="_blank"}, [Amazon DynamoDB](https://aws.amazon.com/dynamodb/){target="_blank"}, or bring your own. ## Key features @@ -479,7 +479,6 @@ Here is the mapping between this utility's functions and methods and the underly | DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb` | ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [query](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.query) | | App Config | `get_app_config` | `appconfig` | [get_configuration](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/appconfig.html#AppConfig.Client.get_configuration) | - ### Bring your own boto client You can use `boto3_client` parameter via any of the available [Provider Classes](#built-in-provider-class). Some providers expect a low level boto3 client while others expect a high level boto3 client, here is the mapping for each of them: @@ -491,7 +490,6 @@ You can use `boto3_client` parameter via any of the available [Provider Classes] | [AppConfigProvider](#appconfigprovider) | low level | `boto3.client("appconfig")` | | [DynamoDBProvider](#dynamodbprovider) | high level | `boto3.resource("dynamodb")` | - Bringing them together in a single code snippet would look like this: ```python title="Example: passing a custom boto3 client for each provider" @@ -571,7 +569,6 @@ The **`config`** , **`boto3_session`**, and **`boto3_client`** parameters enabl ... ``` - ## Testing your code ### Mocking parameter values @@ -645,14 +642,12 @@ object named `get_parameter_mock`. ``` - ### Clearing cache Parameters utility caches all parameter values for performance and cost reasons. However, this can have unintended interference in tests using the same parameter name. Within your tests, you can use `clear_cache` method available in [every provider](#built-in-provider-class). When using multiple providers or higher level functions like `get_parameter`, use `clear_caches` standalone function to clear cache globally. - === "clear_cache method" ```python hl_lines="9" import pytest diff --git a/docs/utilities/parser.md b/docs/utilities/parser.md index c17e2f173c5..8756725d1e0 100644 --- a/docs/utilities/parser.md +++ b/docs/utilities/parser.md @@ -152,20 +152,20 @@ def my_function(): Parser comes with the following built-in models: -| Model name | Description | -| --------------------------------- | ------------------------------------------------------------------ | -| **DynamoDBStreamModel** | Lambda Event Source payload for Amazon DynamoDB Streams | -| **EventBridgeModel** | Lambda Event Source payload for Amazon EventBridge | -| **SqsModel** | Lambda Event Source payload for Amazon SQS | -| **AlbModel** | Lambda Event Source payload for Amazon Application Load Balancer | -| **CloudwatchLogsModel** | Lambda Event Source payload for Amazon CloudWatch Logs | -| **S3Model** | Lambda Event Source payload for Amazon S3 | -| **S3ObjectLambdaEvent** | Lambda Event Source payload for Amazon S3 Object Lambda | -| **KinesisDataStreamModel** | Lambda Event Source payload for Amazon Kinesis Data Streams | -| **SesModel** | Lambda Event Source payload for Amazon Simple Email Service | -| **SnsModel** | Lambda Event Source payload for Amazon Simple Notification Service | -| **APIGatewayProxyEventModel** | Lambda Event Source payload for Amazon API Gateway | -| **APIGatewayProxyEventV2Model** | Lambda Event Source payload for Amazon API Gateway v2 payload | +| Model name | Description | +| ------------------------------- | ------------------------------------------------------------------ | +| **DynamoDBStreamModel** | Lambda Event Source payload for Amazon DynamoDB Streams | +| **EventBridgeModel** | Lambda Event Source payload for Amazon EventBridge | +| **SqsModel** | Lambda Event Source payload for Amazon SQS | +| **AlbModel** | Lambda Event Source payload for Amazon Application Load Balancer | +| **CloudwatchLogsModel** | Lambda Event Source payload for Amazon CloudWatch Logs | +| **S3Model** | Lambda Event Source payload for Amazon S3 | +| **S3ObjectLambdaEvent** | Lambda Event Source payload for Amazon S3 Object Lambda | +| **KinesisDataStreamModel** | Lambda Event Source payload for Amazon Kinesis Data Streams | +| **SesModel** | Lambda Event Source payload for Amazon Simple Email Service | +| **SnsModel** | Lambda Event Source payload for Amazon Simple Notification Service | +| **APIGatewayProxyEventModel** | Lambda Event Source payload for Amazon API Gateway | +| **APIGatewayProxyEventV2Model** | Lambda Event Source payload for Amazon API Gateway v2 payload | ### extending built-in models @@ -174,7 +174,6 @@ You can extend them to include your own models, and yet have all other known fie ???+ tip For Mypy users, we only allow type override for fields where payload is injected e.g. `detail`, `body`, etc. - ```python hl_lines="16-17 28 41" title="Extending EventBridge model as an example" from aws_lambda_powertools.utilities.parser import parse, BaseModel from aws_lambda_powertools.utilities.parser.models import EventBridgeModel @@ -470,7 +469,10 @@ parse(model=UserModel, event=payload) ???+ tip "Tip: Looking to auto-generate models from JSON, YAML, JSON Schemas, OpenApi, etc?" Use Koudai Aono's [data model code generation tool for Pydantic](https://github.com/koxudaxi/datamodel-code-generator) -There are number of advanced use cases well documented in Pydantic's doc such as creating [immutable models](https://pydantic-docs.helpmanual.io/usage/models/#faux-immutability), [declaring fields with dynamic values](https://pydantic-docs.helpmanual.io/usage/models/#field-with-dynamic-default-value)) e.g. UUID, and [helper functions to parse models from files, str](https://pydantic-docs.helpmanual.io/usage/models/#helper-functions), etc. +There are number of advanced use cases well documented in Pydantic's doc such as creating [immutable models](https://pydantic-docs.helpmanual.io/usage/models/#faux-immutability), [declaring fields with dynamic values](https://pydantic-docs.helpmanual.io/usage/models/#field-with-dynamic-default-value). + +???+ tip "Pydantic helper functions" + Pydantic also offers [functions](https://pydantic-docs.helpmanual.io/usage/models/#helper-functions) to parse models from files, dicts, string, etc. Two possible unknown use cases are Models and exception' serialization. Models have methods to [export them](https://pydantic-docs.helpmanual.io/usage/exporting_models/) as `dict`, `JSON`, `JSON Schema`, and Validation exceptions can be exported as JSON. @@ -539,7 +541,7 @@ Artillery load test sample against a [hello world sample](https://github.com/aws ???+ info **Uncompressed package size**: 55M, **p99**: 180.3ms -``` +```javascript Summary report @ 14:36:07(+0200) 2020-10-23 Scenarios launched: 10 Scenarios completed: 10 @@ -562,7 +564,7 @@ Codes: ???+ info **Uncompressed package size**: 128M, **p99**: 193.1ms -``` +```javascript Summary report @ 14:29:23(+0200) 2020-10-23 Scenarios launched: 10 Scenarios completed: 10 diff --git a/docs/utilities/validation.md b/docs/utilities/validation.md index e6ca0841d2d..ec795c99bef 100644 --- a/docs/utilities/validation.md +++ b/docs/utilities/validation.md @@ -134,7 +134,9 @@ Here is a sample custom EventBridge event, where we only validate what's inside --8<-- "docs/shared/validation_basic_jsonschema.py" ``` -This is quite powerful because you can use JMESPath Query language to extract records from [arrays, slice and dice](https://jmespath.org/tutorial.html#list-and-slice-projections), to [pipe expressions](https://jmespath.org/tutorial.html#pipe-expressions) and [function expressions](https://jmespath.org/tutorial.html#functions), where you'd extract what you need before validating the actual payload. +This is quite powerful because you can use JMESPath Query language to extract records from [arrays](https://jmespath.org/tutorial.html#list-and-slice-projections), combine [pipe](https://jmespath.org/tutorial.html#pipe-expressions) and [function expressions](https://jmespath.org/tutorial.html#functions). + +When combined, these features allow you to extract what you need before validating the actual payload. ### Built-in envelopes @@ -166,16 +168,16 @@ This utility comes with built-in envelopes to easily extract the payload from po Here is a handy table with built-in envelopes along with their JMESPath expressions in case you want to build your own. -Envelope name | JMESPath expression -------------------------------------------------- | --------------------------------------------------------------------------------- -**API_GATEWAY_REST** | "powertools_json(body)" -**API_GATEWAY_HTTP** | "powertools_json(body)" -**SQS** | "Records[*].powertools_json(body)" -**SNS** | "Records[0].Sns.Message | powertools_json(@)" -**EVENTBRIDGE** | "detail" -**CLOUDWATCH_EVENTS_SCHEDULED** | "detail" -**KINESIS_DATA_STREAM** | "Records[*].kinesis.powertools_json(powertools_base64(data))" -**CLOUDWATCH_LOGS** | "awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]" +| Envelope name | JMESPath expression | +| ------------------------------- | ------------------------------------------------------------- | +| **API_GATEWAY_REST** | "powertools_json(body)" | +| **API_GATEWAY_HTTP** | "powertools_json(body)" | +| **SQS** | "Records[*].powertools_json(body)" | +| **SNS** | "Records[0].Sns.Message | powertools_json(@)" | +| **EVENTBRIDGE** | "detail" | +| **CLOUDWATCH_EVENTS_SCHEDULED** | "detail" | +| **KINESIS_DATA_STREAM** | "Records[*].kinesis.powertools_json(powertools_base64(data))" | +| **CLOUDWATCH_LOGS** | "awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]" | ## Advanced From 34f842bf35deb68d2ab86704f434f2ef53b1c146 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 22 Jun 2022 18:15:29 +0200 Subject: [PATCH 4/9] docs(lint): include Makefile target for CI --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 0ee0ee76fbd..e2e562ae699 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,12 @@ format: lint: format poetry run flake8 aws_lambda_powertools/* tests/* +lint-docs: + docker run -v ${PWD}:/markdown 06kellyjac/markdownlint-cli:0.28.1-alpine "docs" + +lint-docs-fix: + docker run -v ${PWD}:/markdown 06kellyjac/markdownlint-cli:0.28.1-alpine --fix "docs" + test: poetry run pytest -m "not perf" --cov=aws_lambda_powertools --cov-report=xml poetry run pytest --cache-clear tests/performance From 299fd4f978c3b97657a1cc46e0d0e684b7d5b671 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 22 Jun 2022 18:24:15 +0200 Subject: [PATCH 5/9] docs(lint): sync CI and local pre-commit --- .pre-commit-config.yaml | 59 ++++++++++++++++++++++------------------- Makefile | 4 +-- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 61e98378017..97bdef15726 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,30 +4,35 @@ # All checks can be run locally via `make pr` repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 - hooks: - - id: check-merge-conflict - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-toml - - repo: local - hooks: - - id: black - name: formatting::black - entry: poetry run black - language: system - types: [python] - - id: isort - name: formatting::isort - entry: poetry run isort - language: system - types: [python] - - repo: local - hooks: - - id: flake8 - name: linting::flake8 - entry: poetry run flake8 - language: system - types: [python] - exclude: example + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: check-merge-conflict + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-toml + - repo: local + hooks: + - id: black + name: formatting::black + entry: poetry run black + language: system + types: [python] + - id: isort + name: formatting::isort + entry: poetry run isort + language: system + types: [python] + - repo: local + hooks: + - id: flake8 + name: linting::flake8 + entry: poetry run flake8 + language: system + types: [python] + exclude: example + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: "11c08644ce6df850480d98f628596446a526cbc6" # frozen: v0.31.1 + hooks: + - id: markdownlint + args: ["--fix"] diff --git a/Makefile b/Makefile index e2e562ae699..b4c6b864527 100644 --- a/Makefile +++ b/Makefile @@ -17,10 +17,10 @@ lint: format poetry run flake8 aws_lambda_powertools/* tests/* lint-docs: - docker run -v ${PWD}:/markdown 06kellyjac/markdownlint-cli:0.28.1-alpine "docs" + docker run -v ${PWD}:/markdown 06kellyjac/markdownlint-cli "docs" lint-docs-fix: - docker run -v ${PWD}:/markdown 06kellyjac/markdownlint-cli:0.28.1-alpine --fix "docs" + docker run -v ${PWD}:/markdown 06kellyjac/markdownlint-cli --fix "docs" test: poetry run pytest -m "not perf" --cov=aws_lambda_powertools --cov-report=xml From 6f010f0beb2f446904c18662d75472150e063236 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 22 Jun 2022 18:25:35 +0200 Subject: [PATCH 6/9] docs(lint): disable required heading in homepage --- docs/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.md b/docs/index.md index 60352f9d818..cc3d437334e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,6 +3,8 @@ title: Homepage description: AWS Lambda Powertools Python --- + + A suite of utilities for AWS Lambda functions to ease adopting best practices such as tracing, structured logging, custom metrics, idempotency, batching, and more. ???+ tip "Tip: Looking for a quick read through how the core features are used?" From a1df93886affe7e88f627d8ef972a832c8f5e5e7 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 23 Jun 2022 09:12:12 +0200 Subject: [PATCH 7/9] feat(ci): include docs markdown linting Signed-off-by: heitorlessa --- .github/workflows/python_docs.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python_docs.yml b/.github/workflows/python_docs.yml index 295ecb334c8..d168e1ae68a 100644 --- a/.github/workflows/python_docs.yml +++ b/.github/workflows/python_docs.yml @@ -5,9 +5,9 @@ on: branches: - develop paths: - - 'docs/**' - - 'CHANGELOG.md' - - 'mkdocs.yml' + - "docs/**" + - "CHANGELOG.md" + - "mkdocs.yml" jobs: docs: @@ -22,6 +22,8 @@ jobs: python-version: "3.8" - name: Install dependencies run: make dev + - name: Lint documentation + run: make lint-docs - name: Setup doc deploy run: | git config --global user.name Docs deploy From 69953ce692fe904b324bc0a2a86c5cf9ad7e1a7b Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 23 Jun 2022 09:15:03 +0200 Subject: [PATCH 8/9] feat(ci): include docs linting in make pr --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b4c6b864527..5ad35546f95 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ coverage-html: pre-commit: pre-commit run --show-diff-on-failure -pr: lint mypy pre-commit test security-baseline complexity-baseline +pr: lint lint-docs mypy pre-commit test security-baseline complexity-baseline build: pr poetry build From 4e85b88296aad7779fa921776b2a3410c0f95c56 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 23 Jun 2022 09:15:03 +0200 Subject: [PATCH 9/9] feat(ci): include docs linting in make pr --- Makefile | 2 +- docs/core/event_handler/api_gateway.md | 72 +++++++++++++------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index b4c6b864527..5ad35546f95 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ coverage-html: pre-commit: pre-commit run --show-diff-on-failure -pr: lint mypy pre-commit test security-baseline complexity-baseline +pr: lint lint-docs mypy pre-commit test security-baseline complexity-baseline build: pr poetry build diff --git a/docs/core/event_handler/api_gateway.md b/docs/core/event_handler/api_gateway.md index 0ae3647419c..cf99b615a80 100644 --- a/docs/core/event_handler/api_gateway.md +++ b/docs/core/event_handler/api_gateway.md @@ -5,7 +5,7 @@ description: Core utility Event handler for Amazon API Gateway REST and HTTP APIs, and Application Loader Balancer (ALB). -### Key Features +## Key Features * Lightweight routing to reduce boilerplate for API Gateway REST/HTTP API and ALB * Seamless support for CORS, binary and Gzip compression @@ -25,47 +25,47 @@ This is the sample infrastructure for API Gateway we are using for the examples ???+ info "There is no additional permissions or dependencies required to use this utility." ```yaml title="AWS Serverless Application Model (SAM) example" -AWSTemplateFormatVersion: '2010-09-09' +AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 Description: Hello world event handler API Gateway Globals: - Api: - TracingEnabled: true - Cors: # see CORS section - AllowOrigin: "'https://example.com'" - AllowHeaders: "'Content-Type,Authorization,X-Amz-Date'" - MaxAge: "'300'" - BinaryMediaTypes: # see Binary responses section - - '*~1*' # converts to */* for any binary type - Function: - Timeout: 5 - Runtime: python3.8 - Tracing: Active - Environment: - Variables: - LOG_LEVEL: INFO - POWERTOOLS_LOGGER_SAMPLE_RATE: 0.1 - POWERTOOLS_LOGGER_LOG_EVENT: true - POWERTOOLS_METRICS_NAMESPACE: MyServerlessApplication - POWERTOOLS_SERVICE_NAME: my_api-service + Api: + TracingEnabled: true + Cors: # see CORS section + AllowOrigin: "'https://example.com'" + AllowHeaders: "'Content-Type,Authorization,X-Amz-Date'" + MaxAge: "'300'" + BinaryMediaTypes: # see Binary responses section + - "*~1*" # converts to */* for any binary type + Function: + Timeout: 5 + Runtime: python3.8 + Tracing: Active + Environment: + Variables: + LOG_LEVEL: INFO + POWERTOOLS_LOGGER_SAMPLE_RATE: 0.1 + POWERTOOLS_LOGGER_LOG_EVENT: true + POWERTOOLS_METRICS_NAMESPACE: MyServerlessApplication + POWERTOOLS_SERVICE_NAME: my_api-service Resources: - ApiFunction: - Type: AWS::Serverless::Function - Properties: - Handler: app.lambda_handler - CodeUri: api_handler/ - Description: API handler function - Events: - ApiEvent: - Type: Api - Properties: - # NOTE: this is a catch-all rule to simply the documentation. - # explicit routes and methods are recommended for prod instead - # for example, Path: /hello, Method: GET - Path: /{proxy+} # Send requests on any path to the lambda function - Method: ANY # Send requests using any http method to the lambda function + ApiFunction: + Type: AWS::Serverless::Function + Properties: + Handler: app.lambda_handler + CodeUri: api_handler/ + Description: API handler function + Events: + ApiEvent: + Type: Api + Properties: + # NOTE: this is a catch-all rule to simply the documentation. + # explicit routes and methods are recommended for prod instead + # for example, Path: /hello, Method: GET + Path: /{proxy+} # Send requests on any path to the lambda function + Method: ANY # Send requests using any http method to the lambda function ``` ### Event Resolvers