Upgrade pydantic v1 to v2

CHANGELOG.rst

@@ -2,6 +2,14 @@
 Changelog
 =========
 
+Version 0.5.0
+=============
+
+* Feature: Migrated from Pydantic v1 to Pydantic v2
+* Removed dependency on dataclasses-json in favor of Pydantic models
+* Updated the JSON serialization to work with Pydantic v2
+* Updated schema validation to use the new Pydantic v2 API
+
 Version 0.4.7
 =============
 

deployment/scripts/sync_initial_messages.py

@@ -8,11 +8,14 @@
 initial_messages_list = [
     # Diagnostic VMs
     "cad11970efe9b7478300fd04d7cc91c646ca0a792b9cc718650f86e1ccfac73e",  # Initial program
-    "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af",  # DiagVM
+    "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af",  # Old DiagVM Debian 12
+    "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace",  # Current Debian 12 DiagVM
     "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8",  # Legacy Diag VM
     # Volumes like runtimes, data, code, etc
-    "6b8618f5b8913c0f582f1a771a154a556ee3fa3437ef3cf91097819910cf383b",  # Diag VM code volume
-    "f873715dc2feec3833074bd4b8745363a0e0093746b987b4c8191268883b2463",  # Diag VM runtime volume
+    "6b8618f5b8913c0f582f1a771a154a556ee3fa3437ef3cf91097819910cf383b",  # Old Diag VM code volume
+    "f873715dc2feec3833074bd4b8745363a0e0093746b987b4c8191268883b2463",  # Old Diag VM runtime volume
+    "79f19811f8e843f37ff7535f634b89504da3d8f03e1f0af109d1791cf6add7af",  # Diag VM code volume
+    "63f07193e6ee9d207b7d1fcf8286f9aee34e6f12f101d2ec77c1229f92964696",  # Diag VM runtime volume
     "a92c81992e885d7a554fa78e255a5802404b7fdde5fbff20a443ccd13020d139",  # Legacy Diag VM code volume
     "bd79839bf96e595a06da5ac0b6ba51dea6f7e2591bb913deccded04d831d29f4",  # Legacy Diag VM runtime volume
 ]

pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
   "aiohttp-jinja2==1.6",
   "aioipfs~=0.7.1",
   "alembic==1.15.1",
-  "aleph-message==0.6.1",
+  "aleph-message==1.0.0",
   "aleph-nuls2==0.1",
   "aleph-p2p-client @ git+https://github.com/aleph-im/p2p-service-client-python@cbfebb871db94b2ca580e66104a67cd730c5020c",
   "asyncpg==0.30",
@@ -46,7 +46,8 @@ dependencies = [
   "multiaddr==0.0.9",                                                                                                      # for libp2p-stubs
   "orjson>=3.7.7",                                                                                                         # Minimum version for Python 3.11
   "psycopg2-binary==2.9.10",                                                                                               # Note: psycopg3 is not yet supported by SQLAlchemy
-  "pycryptodome==3.22.0",                                                                                                  # for libp2p-stubs
+  "pycryptodome==3.22.0",
+  "pydantic>=2.0.0,<3.0.0",
   "pymultihash==0.8.2",                                                                                                    # for libp2p-stubs
   "pynacl==1.5",
   "pytezos-crypto==3.13.4.1",
@@ -62,6 +63,7 @@ dependencies = [
   "sqlalchemy-utils==0.41.2",
   "substrate-interface==1.7.11",
   "types-aiofiles==24.1.0.20241221",
+  "typing-extensions>=4.6.1",
   "ujson==5.10.0",                                                                                                         # required by aiocache
   "urllib3==2.3",
   "uvloop==0.21",
@@ -158,9 +160,7 @@ dependencies = [
   "isort==5.13.2",
   "check-sdist==0.1.3",
   "sqlalchemy[mypy]==1.4.41",
-  "yamlfix==1.16.1",
-  # because of aleph messages otherwise yamlfix install a too new version
-  "pydantic>=1.10.5,<2.0.0",
+  "yamlfix==1.17.0",
   "pyproject-fmt==2.2.1",
   "types-aiofiles",
   "types-protobuf",

src/aleph/chains/chain_data_service.py

@@ -1,4 +1,5 @@
 import asyncio
+import json
 from typing import Any, Dict, List, Mapping, Optional, Self, Set, Type, Union, cast
 
 import aio_pika.abc
@@ -66,10 +67,12 @@ async def prepare_sync_event_payload(
             protocol=ChainSyncProtocol.ON_CHAIN_SYNC,
             version=1,
             content=OnChainContent(
-                messages=[OnChainMessage.from_orm(message) for message in messages]
+                messages=[
+                    OnChainMessage.model_validate(message) for message in messages
+                ]
             ),
         )
-        archive_content: bytes = archive.json().encode("utf-8")
+        archive_content: bytes = archive.model_dump_json().encode("utf-8")
 
         ipfs_cid = await self.storage_service.add_file(
             session=session, file_content=archive_content, engine=ItemType.ipfs
@@ -166,7 +169,9 @@ def _get_tx_messages_smart_contract_protocol(tx: ChainTxDb) -> List[Dict[str, An
         )
 
         try:
-            payload = cast(GenericMessageEvent, payload_model.parse_obj(tx.content))
+            payload = cast(
+                GenericMessageEvent, payload_model.model_validate(tx.content)
+            )
         except ValidationError:
             raise InvalidContent(f"Incompatible tx content for {tx.chain}/{tx.hash}")
 
@@ -189,7 +194,7 @@ def _get_tx_messages_smart_contract_protocol(tx: ChainTxDb) -> List[Dict[str, An
                 item_hash=ItemHash(payload.content),
                 metadata=None,
             )
-            item_content = content.json(exclude_none=True)
+            item_content = json.dumps(content.model_dump(exclude_none=True))
         else:
             item_content = payload.content
 

src/aleph/chains/indexer_reader.py

@@ -84,16 +84,16 @@ def make_events_query(
     if not datetime_range and not block_range:
         raise ValueError("A range of datetimes or blocks must be specified.")
 
-    model: Union[Type[MessageEvent], Type[SyncEvent]]
+    model_fields: List[str]
 
     if event_type == ChainEventType.MESSAGE:
-        model = MessageEvent
         event_type_str = "messageEvents"
+        model_fields = list(MessageEvent.model_fields.keys())
     else:
-        model = SyncEvent
         event_type_str = "syncEvents"
+        model_fields = list(SyncEvent.model_fields.keys())
 
-    fields = "\n".join(model.__fields__.keys())
+    fields = "\n".join(model_fields)
     params: Dict[str, Any] = {
         "blockchain": f'"{blockchain.value}"',
         "limit": limit,
@@ -147,7 +147,7 @@ async def _query(self, query: str, model: Type[T]) -> T:
         response = await self.http_session.post("/", json={"query": query})
         response.raise_for_status()
         response_json = await response.json()
-        return model.parse_obj(response_json)
+        return model.model_validate(response_json)
 
     async def fetch_account_state(
         self,
@@ -196,7 +196,7 @@ def indexer_event_to_chain_tx(
     if isinstance(indexer_event, MessageEvent):
         protocol = ChainSyncProtocol.SMART_CONTRACT
         protocol_version = 1
-        content = indexer_event.dict()
+        content = indexer_event.model_dump()
     else:
         sync_message = aleph_json.loads(indexer_event.message)
 

src/aleph/chains/tezos.py

@@ -162,7 +162,7 @@ async def fetch_messages(
     response.raise_for_status()
     response_json = await response.json()
 
-    return IndexerResponse[IndexerMessageEvent].parse_obj(response_json)
+    return IndexerResponse[IndexerMessageEvent].model_validate(response_json)
 
 
 def indexer_event_to_chain_tx(
@@ -176,7 +176,7 @@ def indexer_event_to_chain_tx(
         publisher=indexer_event.source,
         protocol=ChainSyncProtocol.SMART_CONTRACT,
         protocol_version=1,
-        content=indexer_event.payload.dict(),
+        content=indexer_event.payload.model_dump(),
     )
 
     return chain_tx

src/aleph/db/models/messages.py

@@ -14,7 +14,6 @@
     StoreContent,
 )
 from pydantic import ValidationError
-from pydantic.error_wrappers import ErrorWrapper
 from sqlalchemy import (
     ARRAY,
     TIMESTAMP,
@@ -62,14 +61,14 @@ def validate_message_content(
     content_dict: Dict[str, Any],
 ) -> BaseContent:
     content_type = CONTENT_TYPE_MAP[message_type]
-    content = content_type.parse_obj(content_dict)
+    content = content_type.model_validate(content_dict)
     # Validate that the content time can be converted to datetime. This will
     # raise a ValueError and be caught
     # TODO: move this validation in aleph-message
     try:
         _ = dt.datetime.fromtimestamp(content_dict["time"])
     except ValueError as e:
-        raise ValidationError([ErrorWrapper(e, loc="time")], model=content_type) from e
+        raise ValidationError(str(e)) from e
 
     return content
 

src/aleph/handlers/content/vm.py

@@ -195,7 +195,7 @@ def vm_message_to_db(message: MessageDb) -> VmBaseDb:
 
         if content.on.message:
             vm.message_triggers = [
-                subscription.dict() for subscription in content.on.message
+                subscription.model_dump() for subscription in content.on.message
             ]
 
         vm.code_volume = CodeVolumeDb(

src/aleph/schemas/api/accounts.py

@@ -1,9 +1,9 @@
 import datetime as dt
 from decimal import Decimal
-from typing import Dict, List, Optional
+from typing import Annotated, Dict, List, Optional
 
 from aleph_message.models import Chain
-from pydantic import BaseModel, Field, validator
+from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, field_validator
 
 from aleph.types.files import FileType
 from aleph.types.sort_order import SortOrder
@@ -16,11 +16,16 @@ class GetAccountQueryParams(BaseModel):
     )
 
 
+FloatDecimal = Annotated[
+    Decimal, PlainSerializer(lambda x: float(x), return_type=float, when_used="json")
+]
+
+
 class GetAccountBalanceResponse(BaseModel):
     address: str
-    balance: Decimal
-    details: Optional[Dict[str, Decimal]]
-    locked_amount: Decimal
+    balance: FloatDecimal
+    details: Optional[Dict[str, FloatDecimal]] = None
+    locked_amount: FloatDecimal
 
 
 class GetAccountFilesQueryParams(BaseModel):
@@ -53,16 +58,15 @@ class GetBalancesChainsQueryParams(BaseModel):
     )
     min_balance: int = Field(default=0, ge=1, description="Minimum Balance needed")
 
-    @validator("chains", pre=True)
+    @field_validator("chains", mode="before")
     def split_str(cls, v):
         if isinstance(v, str):
             return v.split(LIST_FIELD_SEPARATOR)
         return v
 
 
 class AddressBalanceResponse(BaseModel):
-    class Config:
-        orm_mode = True
+    model_config = ConfigDict(from_attributes=True)
 
     address: str
     balance: str
@@ -78,8 +82,7 @@ class GetAccountFilesResponseItem(BaseModel):
 
 
 class GetAccountFilesResponse(BaseModel):
-    class Config:
-        orm_mode = True
+    model_config = ConfigDict(from_attributes=True)
 
     address: str
     total_size: int

src/aleph/schemas/api/costs.py

@@ -1,27 +1,25 @@
 from typing import List
 
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, ConfigDict, field_validator
 
 from aleph.toolkit.costs import format_cost_str
 
 
 class EstimatedCostDetailResponse(BaseModel):
-    class Config:
-        orm_mode = True
+    model_config = ConfigDict(from_attributes=True)
 
     type: str
     name: str
     cost_hold: str
     cost_stream: str
 
-    @validator("cost_hold", "cost_stream")
+    @field_validator("cost_hold", "cost_stream")
     def check_format_price(cls, v):
         return format_cost_str(v)
 
 
 class EstimatedCostsResponse(BaseModel):
-    class Config:
-        orm_mode = True
+    model_config = ConfigDict(from_attributes=True)
 
     required_tokens: float
     payment_type: str