pydantic v2: initial tests

Author: leandrodamascena

.github/workflows/quality_check_temp_pydanticv2.yml

@@ -0,0 +1,82 @@
+name: Code quality temp - Pydanticv2
+
+# PROCESS
+#
+# 1. Install all dependencies and spin off containers for all supported Python versions
+# 2. Run code formatters and linters (various checks) for code standard
+# 3. Run static typing checker for potential bugs
+# 4. Run entire test suite for regressions except end-to-end (unit, functional, performance)
+# 5. Run static analysis (in addition to CodeQL) for common insecure code practices
+# 6. Run complexity baseline to avoid error-prone bugs and keep maintenance lower
+# 7. Collect and report on test coverage
+
+# USAGE
+#
+# Always triggered on new PRs, PR changes and PR merge.
+
+
+on:
+  pull_request:
+    paths:
+      - "aws_lambda_powertools/**"
+      - "tests/**"
+      - "pyproject.toml"
+      - "poetry.lock"
+      - "mypy.ini"
+    branches:
+      - poc/pydanticv2
+  push:
+    paths:
+      - "aws_lambda_powertools/**"
+      - "tests/**"
+      - "pyproject.toml"
+      - "poetry.lock"
+      - "mypy.ini"
+    branches:
+      - poc/pydanticv2
+
+permissions:
+  contents: read
+
+jobs:
+  quality_check:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+    env:
+      PYTHON: "${{ matrix.python-version }}"
+    permissions:
+      contents: read  # checkout code only
+    steps:
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9  # v3.5.3
+      - name: Install poetry
+        run: pipx install poetry
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4.6.1
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "poetry"
+      - name: Removing cfn-lint
+        run: poetry remove cfn-lint
+      - name: Replacing Pydantic v1 with v2
+        run: poetry add "pydantic>=2.0"
+      - name: Install dependencies
+        run: make dev
+      - name: Formatting and Linting
+        run: make lint
+      - name: Static type checking
+        run: make mypy
+      - name: Test with pytest
+        run: make test
+      - name: Security baseline
+        run: make security-baseline
+      - name: Complexity baseline
+        run: make complexity-baseline
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 3.1.4
+        with:
+          file: ./coverage.xml
+          env_vars: PYTHON
+          name: aws-lambda-powertools-python-codecov

aws_lambda_powertools/shared/user_agent.py

@@ -112,7 +112,7 @@ def register_feature_to_session(session, feature):
 def register_feature_to_botocore_session(botocore_session, feature):
     """
     Register the given feature string to the event system of the provided botocore session
-    
+
     Please notice this function is for patching botocore session and is different from
     previous one which is for patching boto3 session
 
@@ -127,7 +127,7 @@ def register_feature_to_botocore_session(botocore_session, feature):
     ------
     AttributeError
         If the provided session does not have an event system.
-        
+
     Examples
     --------
     **register data-masking user-agent to botocore session**
@@ -139,7 +139,7 @@ def register_feature_to_botocore_session(botocore_session, feature):
         >>> session = botocore.session.Session()
         >>> register_feature_to_botocore_session(botocore_session=session, feature="data-masking")
         >>> key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=session)
-    
+
     """
     try:
         botocore_session.register(TARGET_SDK_EVENT, _create_feature_function(feature))

aws_lambda_powertools/utilities/parser/models/apigw.py

@@ -1,7 +1,7 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional, Type, Union
 
-from pydantic import BaseModel, root_validator
+from pydantic import BaseModel, root_validator, validator
 from pydantic.networks import IPvAnyNetwork
 
 from aws_lambda_powertools.utilities.parser.types import Literal
@@ -21,74 +21,79 @@ class ApiGatewayUserCert(BaseModel):
 
 
 class APIGatewayEventIdentity(BaseModel):
-    accessKey: Optional[str]
-    accountId: Optional[str]
-    apiKey: Optional[str]
-    apiKeyId: Optional[str]
-    caller: Optional[str]
-    cognitoAuthenticationProvider: Optional[str]
-    cognitoAuthenticationType: Optional[str]
-    cognitoIdentityId: Optional[str]
-    cognitoIdentityPoolId: Optional[str]
-    principalOrgId: Optional[str]
+    accessKey: Optional[str] = None
+    accountId: Optional[str] = None
+    apiKey: Optional[str] = None
+    apiKeyId: Optional[str] = None
+    caller: Optional[str] = None
+    cognitoAuthenticationProvider: Optional[str] = None
+    cognitoAuthenticationType: Optional[str] = None
+    cognitoIdentityId: Optional[str] = None
+    cognitoIdentityPoolId: Optional[str] = None
+    principalOrgId: Optional[str] = None
     # see #1562, temp workaround until API Gateway fixes it the Test button payload
     # removing it will not be considered a regression in the future
     sourceIp: Union[IPvAnyNetwork, Literal["test-invoke-source-ip"]]
-    user: Optional[str]
-    userAgent: Optional[str]
-    userArn: Optional[str]
-    clientCert: Optional[ApiGatewayUserCert]
+    user: Optional[str] = None
+    userAgent: Optional[str] = None
+    userArn: Optional[str] = None
+    clientCert: Optional[ApiGatewayUserCert] = None
 
 
 class APIGatewayEventAuthorizer(BaseModel):
-    claims: Optional[Dict[str, Any]]
-    scopes: Optional[List[str]]
+    claims: Optional[Dict[str, Any]] = None
+    scopes: Optional[List[str]] = None
 
 
 class APIGatewayEventRequestContext(BaseModel):
     accountId: str
     apiId: str
-    authorizer: Optional[APIGatewayEventAuthorizer]
+    authorizer: Optional[APIGatewayEventAuthorizer] = None
     stage: str
     protocol: str
     identity: APIGatewayEventIdentity
     requestId: str
     requestTime: str
     requestTimeEpoch: datetime
-    resourceId: Optional[str]
+    resourceId: Optional[str] = None
     resourcePath: str
-    domainName: Optional[str]
-    domainPrefix: Optional[str]
-    extendedRequestId: Optional[str]
+    domainName: Optional[str] = None
+    domainPrefix: Optional[str] = None
+    extendedRequestId: Optional[str] = None
     httpMethod: Literal["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
     path: str
-    connectedAt: Optional[datetime]
-    connectionId: Optional[str]
-    eventType: Optional[Literal["CONNECT", "MESSAGE", "DISCONNECT"]]
-    messageDirection: Optional[str]
-    messageId: Optional[str]
-    routeKey: Optional[str]
-    operationName: Optional[str]
-
-    @root_validator(allow_reuse=True)
+    connectedAt: Optional[datetime] = None
+    connectionId: Optional[str] = None
+    eventType: Optional[Literal["CONNECT", "MESSAGE", "DISCONNECT"]] = None
+    messageDirection: Optional[str] = None
+    messageId: Optional[str] = None
+    routeKey: Optional[str] = None
+    operationName: Optional[str] = None
+
+    @root_validator(allow_reuse=True, skip_on_failure=True)
     def check_message_id(cls, values):
         message_id, event_type = values.get("messageId"), values.get("eventType")
         if message_id is not None and event_type != "MESSAGE":
-            raise TypeError("messageId is available only when the `eventType` is `MESSAGE`")
+            raise ValueError("messageId is available only when the `eventType` is `MESSAGE`")
         return values
 
+    @validator("requestTimeEpoch", pre=True)
+    def normalize_timestamp(cls, value):
+        date_utc = datetime.fromtimestamp(int(value) / 1000, tz=timezone.utc)
+        return date_utc
+
 
 class APIGatewayProxyEventModel(BaseModel):
-    version: Optional[str]
+    version: Optional[str] = None
     resource: str
     path: str
     httpMethod: Literal["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
     headers: Dict[str, str]
     multiValueHeaders: Dict[str, List[str]]
-    queryStringParameters: Optional[Dict[str, str]]
-    multiValueQueryStringParameters: Optional[Dict[str, List[str]]]
+    queryStringParameters: Optional[Dict[str, str]] = None
+    multiValueQueryStringParameters: Optional[Dict[str, List[str]]] = None
     requestContext: APIGatewayEventRequestContext
-    pathParameters: Optional[Dict[str, str]]
-    stageVariables: Optional[Dict[str, str]]
+    pathParameters: Optional[Dict[str, str]] = None
+    stageVariables: Optional[Dict[str, str]] = None
     isBase64Encoded: bool
-    body: Optional[Union[str, Type[BaseModel]]]
+    body: Optional[Union[str, Type[BaseModel]]] = None

aws_lambda_powertools/utilities/parser/models/apigwv2.py

@@ -1,7 +1,7 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional, Type, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, validator
 from pydantic.networks import IPvAnyNetwork
 
 from aws_lambda_powertools.utilities.parser.types import Literal
@@ -14,13 +14,13 @@ class RequestContextV2AuthorizerIamCognito(BaseModel):
 
 
 class RequestContextV2AuthorizerIam(BaseModel):
-    accessKey: Optional[str]
-    accountId: Optional[str]
-    callerId: Optional[str]
-    principalOrgId: Optional[str]
-    userArn: Optional[str]
-    userId: Optional[str]
-    cognitoIdentity: Optional[RequestContextV2AuthorizerIamCognito]
+    accessKey: Optional[str] = None
+    accountId: Optional[str] = None
+    callerId: Optional[str] = None
+    principalOrgId: Optional[str] = None
+    userArn: Optional[str] = None
+    userId: Optional[str] = None
+    cognitoIdentity: Optional[RequestContextV2AuthorizerIamCognito] = None
 
 
 class RequestContextV2AuthorizerJwt(BaseModel):
@@ -29,9 +29,9 @@ class RequestContextV2AuthorizerJwt(BaseModel):
 
 
 class RequestContextV2Authorizer(BaseModel):
-    jwt: Optional[RequestContextV2AuthorizerJwt]
-    iam: Optional[RequestContextV2AuthorizerIam]
-    lambda_value: Optional[Dict[str, Any]] = Field(None, alias="lambda")
+    jwt: Optional[RequestContextV2AuthorizerJwt] = None
+    iam: Optional[RequestContextV2AuthorizerIam] = None
+    lambda_value: Union[Dict[str, Any], None] = Field(None, alias="lambda")
 
 
 class RequestContextV2Http(BaseModel):
@@ -45,7 +45,7 @@ class RequestContextV2Http(BaseModel):
 class RequestContextV2(BaseModel):
     accountId: str
     apiId: str
-    authorizer: Optional[RequestContextV2Authorizer]
+    authorizer: Optional[RequestContextV2Authorizer] = None
     domainName: str
     domainPrefix: str
     requestId: str
@@ -55,17 +55,22 @@ class RequestContextV2(BaseModel):
     timeEpoch: datetime
     http: RequestContextV2Http
 
+    @validator("timeEpoch", pre=True)
+    def normalize_timestamp(cls, value):
+        date_utc = datetime.fromtimestamp(int(value) / 1000, tz=timezone.utc)
+        return date_utc
+
 
 class APIGatewayProxyEventV2Model(BaseModel):
     version: str
     routeKey: str
     rawPath: str
     rawQueryString: str
-    cookies: Optional[List[str]]
+    cookies: Optional[List[str]] = None
     headers: Dict[str, str]
-    queryStringParameters: Optional[Dict[str, str]]
-    pathParameters: Optional[Dict[str, str]]
-    stageVariables: Optional[Dict[str, str]]
+    queryStringParameters: Optional[Dict[str, str]] = None
+    pathParameters: Optional[Dict[str, str]] = None
+    stageVariables: Optional[Dict[str, str]] = None
     requestContext: RequestContextV2
-    body: Optional[Union[str, Type[BaseModel]]]
+    body: Optional[Union[str, Type[BaseModel]]] = None
     isBase64Encoded: bool

aws_lambda_powertools/utilities/parser/models/cloudwatch.py

@@ -2,7 +2,7 @@
 import json
 import logging
 import zlib
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Type, Union
 
 from pydantic import BaseModel, Field, validator
@@ -15,6 +15,11 @@ class CloudWatchLogsLogEvent(BaseModel):
     timestamp: datetime
     message: Union[str, Type[BaseModel]]
 
+    @validator("timestamp", pre=True)
+    def coerc_timestamp(cls, value):
+        date_utc = datetime.fromtimestamp(value / 1000, tz=timezone.utc)
+        return date_utc
+
 
 class CloudWatchLogsDecode(BaseModel):
     messageType: str

aws_lambda_powertools/utilities/parser/models/dynamodb.py

@@ -7,10 +7,10 @@
 
 
 class DynamoDBStreamChangedRecordModel(BaseModel):
-    ApproximateCreationDateTime: Optional[date]
+    ApproximateCreationDateTime: Optional[date] = None
     Keys: Dict[str, Dict[str, Any]]
-    NewImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]]
-    OldImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]]
+    NewImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = None
+    OldImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = None
     SequenceNumber: str
     SizeBytes: int
     StreamViewType: Literal["NEW_AND_OLD_IMAGES", "KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE"]
@@ -40,7 +40,7 @@ class DynamoDBStreamRecordModel(BaseModel):
     awsRegion: str
     eventSourceARN: str
     dynamodb: DynamoDBStreamChangedRecordModel
-    userIdentity: Optional[UserIdentity]
+    userIdentity: Optional[UserIdentity] = None
 
 
 class DynamoDBStreamModel(BaseModel):

aws_lambda_powertools/utilities/parser/models/event_bridge.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import List, Optional
+from typing import List, Union
 
 from pydantic import BaseModel, Field
 
@@ -16,4 +16,4 @@ class EventBridgeModel(BaseModel):
     resources: List[str]
     detail_type: str = Field(None, alias="detail-type")
     detail: RawDictOrModel
-    replay_name: Optional[str] = Field(None, alias="replay-name")
+    replay_name: Union[str, None] = Field(None, alias="replay-name")