From 4da24011cc2fdddf6794214b4c3df4782d849563 Mon Sep 17 00:00:00 2001 From: baitcode Date: Wed, 29 Jan 2025 22:43:33 +0300 Subject: [PATCH 01/12] starknet node abi fetch request --- src/dipdup/datasources/starknet_node.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/dipdup/datasources/starknet_node.py b/src/dipdup/datasources/starknet_node.py index 93afd99b3..f17bfcaa2 100644 --- a/src/dipdup/datasources/starknet_node.py +++ b/src/dipdup/datasources/starknet_node.py @@ -1,5 +1,5 @@ import asyncio -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from dipdup.config import HttpConfig from dipdup.config.starknet_node import StarknetNodeDatasourceConfig @@ -70,3 +70,14 @@ async def get_events( chunk_size=self._http_config.batch_size, continuation_token=continuation_token, ) + + async def get_abi(self, address: str) -> dict[str, Any]: + class_at_response = await self.starknetpy.get_class_at(address, block_number='latest') + + parsed_abi = None + if isinstance(class_at_response.abi, str): + parsed_abi = class_at_response.parsed_abi() + else: + parsed_abi = class_at_response.abi() + + return parsed_abi From 19471bf886547af76aabe3137eed765a93518bd4 Mon Sep 17 00:00:00 2001 From: baitcode Date: Thu, 30 Jan 2025 03:02:35 +0300 Subject: [PATCH 02/12] * updated codegen to fetch abis using starnet node provider --- src/dipdup/codegen/starknet.py | 60 ++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/dipdup/codegen/starknet.py b/src/dipdup/codegen/starknet.py index ffb164dbe..e33ad87f7 100644 --- a/src/dipdup/codegen/starknet.py +++ b/src/dipdup/codegen/starknet.py @@ -1,16 +1,72 @@ +import typing as t from pathlib import Path from dipdup.codegen import CodeGenerator from dipdup.config import HandlerConfig +from dipdup.config import StarknetNodeDatasourceConfig +from dipdup.config.starknet_events import StarknetContractConfig +from dipdup.config.starknet_events import StarknetEventsHandlerConfig from dipdup.config.starknet_events import StarknetEventsIndexConfig +from dipdup.datasources import AbiDatasource +from dipdup.exceptions import AbiNotAvailableError +from dipdup.exceptions import ConfigurationError +from dipdup.exceptions import DatasourceError +from dipdup.utils import json_dumps from dipdup.utils import snake_to_pascal +from dipdup.utils import touch class StarknetCodeGenerator(CodeGenerator): kind = 'starknet' - # NOTE: For now ABIs need to be provided manually - async def generate_abis(self) -> None: ... + async def generate_abis(self) -> None: + for index_config in self._config.indexes.values(): + if isinstance(index_config, StarknetEventsIndexConfig): + await self._fetch_abi(index_config) + + async def _fetch_abi(self, index_config: StarknetEventsIndexConfig) -> None: + datasources: list[AbiDatasource[t.Any]] = [ + self._datasources[datasource_config.name] + for datasource_config in index_config.datasources + if isinstance(datasource_config, StarknetNodeDatasourceConfig) + ] + + contracts: list[StarknetContractConfig] = [ + handler_config.contract + for handler_config in index_config.handlers + if isinstance(handler_config, StarknetEventsHandlerConfig) + ] + + if contracts and not datasources: + raise ConfigurationError('No Starknet ABI datasources found') + + for contract in contracts: + abi_path = self._package.abi / contract.module_name / 'abi.json' + + if abi_path.exists(): + continue + + abi_json = None + + for datasource in datasources: + try: + abi_json = await datasource.get_abi(address=contract.address) + break + except DatasourceError as e: + self._logger.warning('Failed to fetch ABI from `%s`: %s', datasource.name, e) + + # TODO(baitcode): Maybe prioritise manual configuration? + if abi_json is None and contract.abi: + abi_json = contract.abi + + if abi_json is None: + raise AbiNotAvailableError( + address=contract.address, + typename=contract.module_name, + ) + + touch(abi_path) + abi_path.write_bytes(json_dumps(abi_json)) async def generate_schemas(self) -> None: from dipdup.abi.cairo import abi_to_jsonschemas From 31fc44036af70e2595ae1353df4c4682d6e25544 Mon Sep 17 00:00:00 2001 From: baitcode Date: Thu, 30 Jan 2025 03:21:23 +0300 Subject: [PATCH 03/12] Updated changelog, improved code a bit --- CHANGELOG.md | 1 + src/dipdup/codegen/starknet.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69452ae87..f2d539e3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Releases prior to 7.0 has been removed from this file to declutter search result ### Other - deps: `tortoise-orm` updated to 0.24.0. +- starknet.node: Added methods for fetching contract ABI's ## [8.2.0rc1] - 2025-01-24 diff --git a/src/dipdup/codegen/starknet.py b/src/dipdup/codegen/starknet.py index e33ad87f7..698f4cdcf 100644 --- a/src/dipdup/codegen/starknet.py +++ b/src/dipdup/codegen/starknet.py @@ -25,19 +25,23 @@ async def generate_abis(self) -> None: await self._fetch_abi(index_config) async def _fetch_abi(self, index_config: StarknetEventsIndexConfig) -> None: - datasources: list[AbiDatasource[t.Any]] = [ - self._datasources[datasource_config.name] - for datasource_config in index_config.datasources - if isinstance(datasource_config, StarknetNodeDatasourceConfig) - ] - contracts: list[StarknetContractConfig] = [ handler_config.contract for handler_config in index_config.handlers if isinstance(handler_config, StarknetEventsHandlerConfig) ] - if contracts and not datasources: + if not contracts: + self._logger.debug('No contract specified. No ABI to fetch.') + return + + datasources: list[AbiDatasource[t.Any]] = [ + self._datasources[datasource_config.name] + for datasource_config in index_config.datasources + if isinstance(datasource_config, StarknetNodeDatasourceConfig) + ] + + if not datasources: raise ConfigurationError('No Starknet ABI datasources found') for contract in contracts: From a9ba848866a46a26fe22e3782b9c618e50047dd8 Mon Sep 17 00:00:00 2001 From: baitcode Date: Thu, 30 Jan 2025 16:15:01 +0300 Subject: [PATCH 04/12] fix mistake --- src/dipdup/datasources/starknet_node.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/dipdup/datasources/starknet_node.py b/src/dipdup/datasources/starknet_node.py index f17bfcaa2..b928ec122 100644 --- a/src/dipdup/datasources/starknet_node.py +++ b/src/dipdup/datasources/starknet_node.py @@ -73,11 +73,10 @@ async def get_events( async def get_abi(self, address: str) -> dict[str, Any]: class_at_response = await self.starknetpy.get_class_at(address, block_number='latest') - parsed_abi = None if isinstance(class_at_response.abi, str): - parsed_abi = class_at_response.parsed_abi() + parsed_abi = class_at_response.parsed_abi else: - parsed_abi = class_at_response.abi() + parsed_abi = class_at_response.abi return parsed_abi From 3a3a5608d3a692d3a6936e483bceb6acaff885fd Mon Sep 17 00:00:00 2001 From: baitcode Date: Fri, 31 Jan 2025 05:26:23 +0300 Subject: [PATCH 05/12] deleted template fixed linter --- .../abi/stark_usdt/cairo_abi.json | 1029 ----------------- src/dipdup/codegen/starknet.py | 23 +- src/dipdup/datasources/starknet_node.py | 27 +- .../abi/stark_usdt/cairo_abi.json.j2 | 1029 ----------------- 4 files changed, 30 insertions(+), 2078 deletions(-) delete mode 100644 src/demo_starknet_events/abi/stark_usdt/cairo_abi.json delete mode 100644 src/dipdup/projects/demo_starknet_events/abi/stark_usdt/cairo_abi.json.j2 diff --git a/src/demo_starknet_events/abi/stark_usdt/cairo_abi.json b/src/demo_starknet_events/abi/stark_usdt/cairo_abi.json deleted file mode 100644 index 2f66499af..000000000 --- a/src/demo_starknet_events/abi/stark_usdt/cairo_abi.json +++ /dev/null @@ -1,1029 +0,0 @@ -[ - { - "interface_name": "src::mintable_token_interface::IMintableToken", - "name": "MintableToken", - "type": "impl" - }, - { - "members": [ - { - "name": "low", - "type": "core::integer::u128" - }, - { - "name": "high", - "type": "core::integer::u128" - } - ], - "name": "core::integer::u256", - "type": "struct" - }, - { - "items": [ - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "permissioned_mint", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "permissioned_burn", - "outputs": [], - "state_mutability": "external", - "type": "function" - } - ], - "name": "src::mintable_token_interface::IMintableToken", - "type": "interface" - }, - { - "interface_name": "src::mintable_token_interface::IMintableTokenCamel", - "name": "MintableTokenCamelImpl", - "type": "impl" - }, - { - "items": [ - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "permissionedMint", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "permissionedBurn", - "outputs": [], - "state_mutability": "external", - "type": "function" - } - ], - "name": "src::mintable_token_interface::IMintableTokenCamel", - "type": "interface" - }, - { - "interface_name": "src::replaceability_interface::IReplaceable", - "name": "Replaceable", - "type": "impl" - }, - { - "members": [ - { - "name": "snapshot", - "type": "@core::array::Array::" - } - ], - "name": "core::array::Span::", - "type": "struct" - }, - { - "members": [ - { - "name": "eic_hash", - "type": "core::starknet::class_hash::ClassHash" - }, - { - "name": "eic_init_data", - "type": "core::array::Span::" - } - ], - "name": "src::replaceability_interface::EICData", - "type": "struct" - }, - { - "name": "core::option::Option::", - "type": "enum", - "variants": [ - { - "name": "Some", - "type": "src::replaceability_interface::EICData" - }, - { - "name": "None", - "type": "()" - } - ] - }, - { - "name": "core::bool", - "type": "enum", - "variants": [ - { - "name": "False", - "type": "()" - }, - { - "name": "True", - "type": "()" - } - ] - }, - { - "members": [ - { - "name": "impl_hash", - "type": "core::starknet::class_hash::ClassHash" - }, - { - "name": "eic_data", - "type": "core::option::Option::" - }, - { - "name": "final", - "type": "core::bool" - } - ], - "name": "src::replaceability_interface::ImplementationData", - "type": "struct" - }, - { - "items": [ - { - "inputs": [], - "name": "get_upgrade_delay", - "outputs": [ - { - "type": "core::integer::u64" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "get_impl_activation_time", - "outputs": [ - { - "type": "core::integer::u64" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "add_new_implementation", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "remove_implementation", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "replace_to", - "outputs": [], - "state_mutability": "external", - "type": "function" - } - ], - "name": "src::replaceability_interface::IReplaceable", - "type": "interface" - }, - { - "interface_name": "src::access_control_interface::IAccessControl", - "name": "AccessControlImplExternal", - "type": "impl" - }, - { - "items": [ - { - "inputs": [ - { - "name": "role", - "type": "core::felt252" - }, - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "has_role", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "role", - "type": "core::felt252" - } - ], - "name": "get_role_admin", - "outputs": [ - { - "type": "core::felt252" - } - ], - "state_mutability": "view", - "type": "function" - } - ], - "name": "src::access_control_interface::IAccessControl", - "type": "interface" - }, - { - "interface_name": "src::roles_interface::IMinimalRoles", - "name": "RolesImpl", - "type": "impl" - }, - { - "items": [ - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "is_governance_admin", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "is_upgrade_governor", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "register_governance_admin", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "remove_governance_admin", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "register_upgrade_governor", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "remove_upgrade_governor", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "role", - "type": "core::felt252" - } - ], - "name": "renounce", - "outputs": [], - "state_mutability": "external", - "type": "function" - } - ], - "name": "src::roles_interface::IMinimalRoles", - "type": "interface" - }, - { - "interface_name": "openzeppelin::token::erc20::interface::IERC20", - "name": "ERC20Impl", - "type": "impl" - }, - { - "items": [ - { - "inputs": [], - "name": "name", - "outputs": [ - { - "type": "core::felt252" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "symbol", - "outputs": [ - { - "type": "core::felt252" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "decimals", - "outputs": [ - { - "type": "core::integer::u8" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "total_supply", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "balance_of", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "owner", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "allowance", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "recipient", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "transfer", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "sender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "recipient", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "transfer_from", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "approve", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - } - ], - "name": "openzeppelin::token::erc20::interface::IERC20", - "type": "interface" - }, - { - "interface_name": "openzeppelin::token::erc20::interface::IERC20CamelOnly", - "name": "ERC20CamelOnlyImpl", - "type": "impl" - }, - { - "items": [ - { - "inputs": [], - "name": "totalSupply", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "balanceOf", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "sender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "recipient", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "transferFrom", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - } - ], - "name": "openzeppelin::token::erc20::interface::IERC20CamelOnly", - "type": "interface" - }, - { - "inputs": [ - { - "name": "name", - "type": "core::felt252" - }, - { - "name": "symbol", - "type": "core::felt252" - }, - { - "name": "decimals", - "type": "core::integer::u8" - }, - { - "name": "initial_supply", - "type": "core::integer::u256" - }, - { - "name": "recipient", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "permitted_minter", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "provisional_governance_admin", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "upgrade_delay", - "type": "core::integer::u64" - } - ], - "name": "constructor", - "type": "constructor" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "added_value", - "type": "core::integer::u256" - } - ], - "name": "increase_allowance", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "subtracted_value", - "type": "core::integer::u256" - } - ], - "name": "decrease_allowance", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "addedValue", - "type": "core::integer::u256" - } - ], - "name": "increaseAllowance", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "subtractedValue", - "type": "core::integer::u256" - } - ], - "name": "decreaseAllowance", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "from", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "to", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "value", - "type": "core::integer::u256" - } - ], - "name": "openzeppelin::token::erc20_v070::erc20::ERC20::Transfer", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "owner", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "value", - "type": "core::integer::u256" - } - ], - "name": "openzeppelin::token::erc20_v070::erc20::ERC20::Approval", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "src::replaceability_interface::ImplementationAdded", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "src::replaceability_interface::ImplementationRemoved", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "src::replaceability_interface::ImplementationReplaced", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "impl_hash", - "type": "core::starknet::class_hash::ClassHash" - } - ], - "name": "src::replaceability_interface::ImplementationFinalized", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "role", - "type": "core::felt252" - }, - { - "kind": "data", - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "sender", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::access_control_interface::RoleGranted", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "role", - "type": "core::felt252" - }, - { - "kind": "data", - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "sender", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::access_control_interface::RoleRevoked", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "role", - "type": "core::felt252" - }, - { - "kind": "data", - "name": "previous_admin_role", - "type": "core::felt252" - }, - { - "kind": "data", - "name": "new_admin_role", - "type": "core::felt252" - } - ], - "name": "src::access_control_interface::RoleAdminChanged", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "added_account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "added_by", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::roles_interface::GovernanceAdminAdded", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "removed_account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "removed_by", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::roles_interface::GovernanceAdminRemoved", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "added_account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "added_by", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::roles_interface::UpgradeGovernorAdded", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "removed_account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "removed_by", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::roles_interface::UpgradeGovernorRemoved", - "type": "event" - }, - { - "kind": "enum", - "name": "openzeppelin::token::erc20_v070::erc20::ERC20::Event", - "type": "event", - "variants": [ - { - "kind": "nested", - "name": "Transfer", - "type": "openzeppelin::token::erc20_v070::erc20::ERC20::Transfer" - }, - { - "kind": "nested", - "name": "Approval", - "type": "openzeppelin::token::erc20_v070::erc20::ERC20::Approval" - }, - { - "kind": "nested", - "name": "ImplementationAdded", - "type": "src::replaceability_interface::ImplementationAdded" - }, - { - "kind": "nested", - "name": "ImplementationRemoved", - "type": "src::replaceability_interface::ImplementationRemoved" - }, - { - "kind": "nested", - "name": "ImplementationReplaced", - "type": "src::replaceability_interface::ImplementationReplaced" - }, - { - "kind": "nested", - "name": "ImplementationFinalized", - "type": "src::replaceability_interface::ImplementationFinalized" - }, - { - "kind": "nested", - "name": "RoleGranted", - "type": "src::access_control_interface::RoleGranted" - }, - { - "kind": "nested", - "name": "RoleRevoked", - "type": "src::access_control_interface::RoleRevoked" - }, - { - "kind": "nested", - "name": "RoleAdminChanged", - "type": "src::access_control_interface::RoleAdminChanged" - }, - { - "kind": "nested", - "name": "GovernanceAdminAdded", - "type": "src::roles_interface::GovernanceAdminAdded" - }, - { - "kind": "nested", - "name": "GovernanceAdminRemoved", - "type": "src::roles_interface::GovernanceAdminRemoved" - }, - { - "kind": "nested", - "name": "UpgradeGovernorAdded", - "type": "src::roles_interface::UpgradeGovernorAdded" - }, - { - "kind": "nested", - "name": "UpgradeGovernorRemoved", - "type": "src::roles_interface::UpgradeGovernorRemoved" - } - ] - } -] \ No newline at end of file diff --git a/src/dipdup/codegen/starknet.py b/src/dipdup/codegen/starknet.py index 698f4cdcf..25d875723 100644 --- a/src/dipdup/codegen/starknet.py +++ b/src/dipdup/codegen/starknet.py @@ -1,12 +1,13 @@ -import typing as t from pathlib import Path +from typing import Any +from typing import cast from dipdup.codegen import CodeGenerator from dipdup.config import HandlerConfig -from dipdup.config import StarknetNodeDatasourceConfig -from dipdup.config.starknet_events import StarknetContractConfig +from dipdup.config.starknet import StarknetContractConfig from dipdup.config.starknet_events import StarknetEventsHandlerConfig from dipdup.config.starknet_events import StarknetEventsIndexConfig +from dipdup.config.starknet_node import StarknetNodeDatasourceConfig from dipdup.datasources import AbiDatasource from dipdup.exceptions import AbiNotAvailableError from dipdup.exceptions import ConfigurationError @@ -35,8 +36,8 @@ async def _fetch_abi(self, index_config: StarknetEventsIndexConfig) -> None: self._logger.debug('No contract specified. No ABI to fetch.') return - datasources: list[AbiDatasource[t.Any]] = [ - self._datasources[datasource_config.name] + datasources: list[AbiDatasource[Any]] = [ + cast(AbiDatasource[Any], self._datasources[datasource_config.name]) for datasource_config in index_config.datasources if isinstance(datasource_config, StarknetNodeDatasourceConfig) ] @@ -52,20 +53,20 @@ async def _fetch_abi(self, index_config: StarknetEventsIndexConfig) -> None: abi_json = None + address = contract.address or contract.abi + if not address: + raise ConfigurationError(f'`address` or `abi` must be specified for contract `{contract.module_name}`') + for datasource in datasources: try: - abi_json = await datasource.get_abi(address=contract.address) + abi_json = await datasource.get_abi(address=address) break except DatasourceError as e: self._logger.warning('Failed to fetch ABI from `%s`: %s', datasource.name, e) - # TODO(baitcode): Maybe prioritise manual configuration? - if abi_json is None and contract.abi: - abi_json = contract.abi - if abi_json is None: raise AbiNotAvailableError( - address=contract.address, + address=address, typename=contract.module_name, ) diff --git a/src/dipdup/datasources/starknet_node.py b/src/dipdup/datasources/starknet_node.py index b928ec122..419aa6a3a 100644 --- a/src/dipdup/datasources/starknet_node.py +++ b/src/dipdup/datasources/starknet_node.py @@ -1,14 +1,20 @@ import asyncio -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING +from typing import Any + +from starknet_py.net.client_models import DeprecatedContractClass +from starknet_py.net.client_models import EventsChunk +from starknet_py.net.client_models import SierraContractClass from dipdup.config import HttpConfig from dipdup.config.starknet_node import StarknetNodeDatasourceConfig from dipdup.datasources import IndexDatasource +from dipdup.datasources._starknetpy import StarknetpyClient if TYPE_CHECKING: - from starknet_py.net.client_models import EventsChunk - - from dipdup.datasources._starknetpy import StarknetpyClient + from starknet_py.abi.v0.shape import AbiDictList as AbiDictListV0 + from starknet_py.abi.v1.shape import AbiDictList as AbiDictListV1 + from starknet_py.abi.v2.shape import AbiDictList as AbiDictListV2 class StarknetNodeDatasource(IndexDatasource[StarknetNodeDatasourceConfig]): @@ -70,13 +76,16 @@ async def get_events( chunk_size=self._http_config.batch_size, continuation_token=continuation_token, ) - + async def get_abi(self, address: str) -> dict[str, Any]: class_at_response = await self.starknetpy.get_class_at(address, block_number='latest') - parsed_abi = None - if isinstance(class_at_response.abi, str): + # NOTE: for some reason + parsed_abi: AbiDictListV0 | None | AbiDictListV1 | AbiDictListV2 + if isinstance(class_at_response, SierraContractClass): parsed_abi = class_at_response.parsed_abi - else: + elif isinstance(class_at_response, DeprecatedContractClass): parsed_abi = class_at_response.abi + else: + raise NotImplementedError(f'Unknown response class: {class_at_response}') - return parsed_abi + return {'starknet_py_abi': parsed_abi} diff --git a/src/dipdup/projects/demo_starknet_events/abi/stark_usdt/cairo_abi.json.j2 b/src/dipdup/projects/demo_starknet_events/abi/stark_usdt/cairo_abi.json.j2 deleted file mode 100644 index 2f66499af..000000000 --- a/src/dipdup/projects/demo_starknet_events/abi/stark_usdt/cairo_abi.json.j2 +++ /dev/null @@ -1,1029 +0,0 @@ -[ - { - "interface_name": "src::mintable_token_interface::IMintableToken", - "name": "MintableToken", - "type": "impl" - }, - { - "members": [ - { - "name": "low", - "type": "core::integer::u128" - }, - { - "name": "high", - "type": "core::integer::u128" - } - ], - "name": "core::integer::u256", - "type": "struct" - }, - { - "items": [ - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "permissioned_mint", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "permissioned_burn", - "outputs": [], - "state_mutability": "external", - "type": "function" - } - ], - "name": "src::mintable_token_interface::IMintableToken", - "type": "interface" - }, - { - "interface_name": "src::mintable_token_interface::IMintableTokenCamel", - "name": "MintableTokenCamelImpl", - "type": "impl" - }, - { - "items": [ - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "permissionedMint", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "permissionedBurn", - "outputs": [], - "state_mutability": "external", - "type": "function" - } - ], - "name": "src::mintable_token_interface::IMintableTokenCamel", - "type": "interface" - }, - { - "interface_name": "src::replaceability_interface::IReplaceable", - "name": "Replaceable", - "type": "impl" - }, - { - "members": [ - { - "name": "snapshot", - "type": "@core::array::Array::" - } - ], - "name": "core::array::Span::", - "type": "struct" - }, - { - "members": [ - { - "name": "eic_hash", - "type": "core::starknet::class_hash::ClassHash" - }, - { - "name": "eic_init_data", - "type": "core::array::Span::" - } - ], - "name": "src::replaceability_interface::EICData", - "type": "struct" - }, - { - "name": "core::option::Option::", - "type": "enum", - "variants": [ - { - "name": "Some", - "type": "src::replaceability_interface::EICData" - }, - { - "name": "None", - "type": "()" - } - ] - }, - { - "name": "core::bool", - "type": "enum", - "variants": [ - { - "name": "False", - "type": "()" - }, - { - "name": "True", - "type": "()" - } - ] - }, - { - "members": [ - { - "name": "impl_hash", - "type": "core::starknet::class_hash::ClassHash" - }, - { - "name": "eic_data", - "type": "core::option::Option::" - }, - { - "name": "final", - "type": "core::bool" - } - ], - "name": "src::replaceability_interface::ImplementationData", - "type": "struct" - }, - { - "items": [ - { - "inputs": [], - "name": "get_upgrade_delay", - "outputs": [ - { - "type": "core::integer::u64" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "get_impl_activation_time", - "outputs": [ - { - "type": "core::integer::u64" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "add_new_implementation", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "remove_implementation", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "replace_to", - "outputs": [], - "state_mutability": "external", - "type": "function" - } - ], - "name": "src::replaceability_interface::IReplaceable", - "type": "interface" - }, - { - "interface_name": "src::access_control_interface::IAccessControl", - "name": "AccessControlImplExternal", - "type": "impl" - }, - { - "items": [ - { - "inputs": [ - { - "name": "role", - "type": "core::felt252" - }, - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "has_role", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "role", - "type": "core::felt252" - } - ], - "name": "get_role_admin", - "outputs": [ - { - "type": "core::felt252" - } - ], - "state_mutability": "view", - "type": "function" - } - ], - "name": "src::access_control_interface::IAccessControl", - "type": "interface" - }, - { - "interface_name": "src::roles_interface::IMinimalRoles", - "name": "RolesImpl", - "type": "impl" - }, - { - "items": [ - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "is_governance_admin", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "is_upgrade_governor", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "register_governance_admin", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "remove_governance_admin", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "register_upgrade_governor", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "remove_upgrade_governor", - "outputs": [], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "role", - "type": "core::felt252" - } - ], - "name": "renounce", - "outputs": [], - "state_mutability": "external", - "type": "function" - } - ], - "name": "src::roles_interface::IMinimalRoles", - "type": "interface" - }, - { - "interface_name": "openzeppelin::token::erc20::interface::IERC20", - "name": "ERC20Impl", - "type": "impl" - }, - { - "items": [ - { - "inputs": [], - "name": "name", - "outputs": [ - { - "type": "core::felt252" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "symbol", - "outputs": [ - { - "type": "core::felt252" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "decimals", - "outputs": [ - { - "type": "core::integer::u8" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "total_supply", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "balance_of", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "owner", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "allowance", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "recipient", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "transfer", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "sender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "recipient", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "transfer_from", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "approve", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - } - ], - "name": "openzeppelin::token::erc20::interface::IERC20", - "type": "interface" - }, - { - "interface_name": "openzeppelin::token::erc20::interface::IERC20CamelOnly", - "name": "ERC20CamelOnlyImpl", - "type": "impl" - }, - { - "items": [ - { - "inputs": [], - "name": "totalSupply", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "balanceOf", - "outputs": [ - { - "type": "core::integer::u256" - } - ], - "state_mutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "name": "sender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "recipient", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "amount", - "type": "core::integer::u256" - } - ], - "name": "transferFrom", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - } - ], - "name": "openzeppelin::token::erc20::interface::IERC20CamelOnly", - "type": "interface" - }, - { - "inputs": [ - { - "name": "name", - "type": "core::felt252" - }, - { - "name": "symbol", - "type": "core::felt252" - }, - { - "name": "decimals", - "type": "core::integer::u8" - }, - { - "name": "initial_supply", - "type": "core::integer::u256" - }, - { - "name": "recipient", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "permitted_minter", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "provisional_governance_admin", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "upgrade_delay", - "type": "core::integer::u64" - } - ], - "name": "constructor", - "type": "constructor" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "added_value", - "type": "core::integer::u256" - } - ], - "name": "increase_allowance", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "subtracted_value", - "type": "core::integer::u256" - } - ], - "name": "decrease_allowance", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "addedValue", - "type": "core::integer::u256" - } - ], - "name": "increaseAllowance", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "inputs": [ - { - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "name": "subtractedValue", - "type": "core::integer::u256" - } - ], - "name": "decreaseAllowance", - "outputs": [ - { - "type": "core::bool" - } - ], - "state_mutability": "external", - "type": "function" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "from", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "to", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "value", - "type": "core::integer::u256" - } - ], - "name": "openzeppelin::token::erc20_v070::erc20::ERC20::Transfer", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "owner", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "spender", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "value", - "type": "core::integer::u256" - } - ], - "name": "openzeppelin::token::erc20_v070::erc20::ERC20::Approval", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "src::replaceability_interface::ImplementationAdded", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "src::replaceability_interface::ImplementationRemoved", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "implementation_data", - "type": "src::replaceability_interface::ImplementationData" - } - ], - "name": "src::replaceability_interface::ImplementationReplaced", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "impl_hash", - "type": "core::starknet::class_hash::ClassHash" - } - ], - "name": "src::replaceability_interface::ImplementationFinalized", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "role", - "type": "core::felt252" - }, - { - "kind": "data", - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "sender", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::access_control_interface::RoleGranted", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "role", - "type": "core::felt252" - }, - { - "kind": "data", - "name": "account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "sender", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::access_control_interface::RoleRevoked", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "role", - "type": "core::felt252" - }, - { - "kind": "data", - "name": "previous_admin_role", - "type": "core::felt252" - }, - { - "kind": "data", - "name": "new_admin_role", - "type": "core::felt252" - } - ], - "name": "src::access_control_interface::RoleAdminChanged", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "added_account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "added_by", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::roles_interface::GovernanceAdminAdded", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "removed_account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "removed_by", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::roles_interface::GovernanceAdminRemoved", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "added_account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "added_by", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::roles_interface::UpgradeGovernorAdded", - "type": "event" - }, - { - "kind": "struct", - "members": [ - { - "kind": "data", - "name": "removed_account", - "type": "core::starknet::contract_address::ContractAddress" - }, - { - "kind": "data", - "name": "removed_by", - "type": "core::starknet::contract_address::ContractAddress" - } - ], - "name": "src::roles_interface::UpgradeGovernorRemoved", - "type": "event" - }, - { - "kind": "enum", - "name": "openzeppelin::token::erc20_v070::erc20::ERC20::Event", - "type": "event", - "variants": [ - { - "kind": "nested", - "name": "Transfer", - "type": "openzeppelin::token::erc20_v070::erc20::ERC20::Transfer" - }, - { - "kind": "nested", - "name": "Approval", - "type": "openzeppelin::token::erc20_v070::erc20::ERC20::Approval" - }, - { - "kind": "nested", - "name": "ImplementationAdded", - "type": "src::replaceability_interface::ImplementationAdded" - }, - { - "kind": "nested", - "name": "ImplementationRemoved", - "type": "src::replaceability_interface::ImplementationRemoved" - }, - { - "kind": "nested", - "name": "ImplementationReplaced", - "type": "src::replaceability_interface::ImplementationReplaced" - }, - { - "kind": "nested", - "name": "ImplementationFinalized", - "type": "src::replaceability_interface::ImplementationFinalized" - }, - { - "kind": "nested", - "name": "RoleGranted", - "type": "src::access_control_interface::RoleGranted" - }, - { - "kind": "nested", - "name": "RoleRevoked", - "type": "src::access_control_interface::RoleRevoked" - }, - { - "kind": "nested", - "name": "RoleAdminChanged", - "type": "src::access_control_interface::RoleAdminChanged" - }, - { - "kind": "nested", - "name": "GovernanceAdminAdded", - "type": "src::roles_interface::GovernanceAdminAdded" - }, - { - "kind": "nested", - "name": "GovernanceAdminRemoved", - "type": "src::roles_interface::GovernanceAdminRemoved" - }, - { - "kind": "nested", - "name": "UpgradeGovernorAdded", - "type": "src::roles_interface::UpgradeGovernorAdded" - }, - { - "kind": "nested", - "name": "UpgradeGovernorRemoved", - "type": "src::roles_interface::UpgradeGovernorRemoved" - } - ] - } -] \ No newline at end of file From 8a5017fcb9b8fea745444883388201aef2750e58 Mon Sep 17 00:00:00 2001 From: baitcode Date: Sat, 1 Feb 2025 19:10:18 +0300 Subject: [PATCH 06/12] codegen research --- src/dipdup/abi/evm.py | 8 ++++++-- src/dipdup/codegen/starknet.py | 4 ++-- src/dipdup/datasources/starknet_node.py | 2 +- src/dipdup/fetcher.py | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/dipdup/abi/evm.py b/src/dipdup/abi/evm.py index 21f66fc2e..cdc5c2ad4 100644 --- a/src/dipdup/abi/evm.py +++ b/src/dipdup/abi/evm.py @@ -96,8 +96,12 @@ def convert_abi(package: DipDupPackage) -> dict[str, EvmAbi]: abi_by_typename: dict[str, EvmAbi] = {} for abi_path in package.evm_abi_paths: - converted_abi = _convert_abi(abi_path) - abi_by_typename[abi_path.parent.stem] = converted_abi + try: + converted_abi = _convert_abi(abi_path) + abi_by_typename[abi_path.parent.stem] = converted_abi + except KeyError as e: + _logger.error('Evm Abi manager can not decod abi file %s', abi_path, exc_info=e) + pass return abi_by_typename diff --git a/src/dipdup/codegen/starknet.py b/src/dipdup/codegen/starknet.py index 25d875723..2c6dc210d 100644 --- a/src/dipdup/codegen/starknet.py +++ b/src/dipdup/codegen/starknet.py @@ -46,7 +46,7 @@ async def _fetch_abi(self, index_config: StarknetEventsIndexConfig) -> None: raise ConfigurationError('No Starknet ABI datasources found') for contract in contracts: - abi_path = self._package.abi / contract.module_name / 'abi.json' + abi_path = self._package.abi / contract.module_name / 'cairo_abi.json' if abi_path.exists(): continue @@ -71,7 +71,7 @@ async def _fetch_abi(self, index_config: StarknetEventsIndexConfig) -> None: ) touch(abi_path) - abi_path.write_bytes(json_dumps(abi_json)) + abi_path.write_bytes(json_dumps(abi_json['cairo_abi'])) async def generate_schemas(self) -> None: from dipdup.abi.cairo import abi_to_jsonschemas diff --git a/src/dipdup/datasources/starknet_node.py b/src/dipdup/datasources/starknet_node.py index 419aa6a3a..e01f5a262 100644 --- a/src/dipdup/datasources/starknet_node.py +++ b/src/dipdup/datasources/starknet_node.py @@ -88,4 +88,4 @@ async def get_abi(self, address: str) -> dict[str, Any]: else: raise NotImplementedError(f'Unknown response class: {class_at_response}') - return {'starknet_py_abi': parsed_abi} + return {'cairo_abi': parsed_abi} diff --git a/src/dipdup/fetcher.py b/src/dipdup/fetcher.py index 26ad666de..71556cb35 100644 --- a/src/dipdup/fetcher.py +++ b/src/dipdup/fetcher.py @@ -42,7 +42,7 @@ def block_number(self) -> int: async def yield_by_level( - iterable: AsyncIterator[tuple[BufferT, ...]] + iterable: AsyncIterator[tuple[BufferT, ...]], ) -> AsyncGenerator[tuple[Level, tuple[BufferT, ...]], None]: items: tuple[BufferT, ...] = () From 32410d644a21ee5283e4a00631725a45d426d3a5 Mon Sep 17 00:00:00 2001 From: baitcode Date: Sat, 1 Feb 2025 20:54:15 +0300 Subject: [PATCH 07/12] Fixed abi loading (tuned out to be caching issue) --- src/dipdup/abi/cairo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dipdup/abi/cairo.py b/src/dipdup/abi/cairo.py index 7f0d92c52..95ec1097f 100644 --- a/src/dipdup/abi/cairo.py +++ b/src/dipdup/abi/cairo.py @@ -74,7 +74,6 @@ def sn_keccak(x: str) -> str: return f'0x{int.from_bytes(keccak_hash, "big") & (1 << 250) - 1:x}' -@cache def _loaded_abis(package: DipDupPackage) -> dict[str, Abi]: from starknet_py.abi.v2.parser import AbiParser From 796ecd744301eecc7a8d8090a787151209883077 Mon Sep 17 00:00:00 2001 From: baitcode Date: Sat, 1 Feb 2025 21:11:35 +0300 Subject: [PATCH 08/12] moved starknetpy imports closer to usage --- src/dipdup/datasources/starknet_node.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dipdup/datasources/starknet_node.py b/src/dipdup/datasources/starknet_node.py index e01f5a262..ae963157d 100644 --- a/src/dipdup/datasources/starknet_node.py +++ b/src/dipdup/datasources/starknet_node.py @@ -2,10 +2,6 @@ from typing import TYPE_CHECKING from typing import Any -from starknet_py.net.client_models import DeprecatedContractClass -from starknet_py.net.client_models import EventsChunk -from starknet_py.net.client_models import SierraContractClass - from dipdup.config import HttpConfig from dipdup.config.starknet_node import StarknetNodeDatasourceConfig from dipdup.datasources import IndexDatasource @@ -15,6 +11,7 @@ from starknet_py.abi.v0.shape import AbiDictList as AbiDictListV0 from starknet_py.abi.v1.shape import AbiDictList as AbiDictListV1 from starknet_py.abi.v2.shape import AbiDictList as AbiDictListV2 + from starknet_py.net.client_models import EventsChunk class StarknetNodeDatasource(IndexDatasource[StarknetNodeDatasourceConfig]): @@ -78,6 +75,9 @@ async def get_events( ) async def get_abi(self, address: str) -> dict[str, Any]: + from starknet_py.net.client_models import DeprecatedContractClass + from starknet_py.net.client_models import SierraContractClass + class_at_response = await self.starknetpy.get_class_at(address, block_number='latest') # NOTE: for some reason parsed_abi: AbiDictListV0 | None | AbiDictListV1 | AbiDictListV2 From 43a266a83fd06b269e36f37c460a74301c06b8e9 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Sun, 2 Feb 2025 13:23:32 -0300 Subject: [PATCH 09/12] Run `make demos` --- .../abi/stark_usdt/cairo_abi.json | 1029 +++++++++++++++++ 1 file changed, 1029 insertions(+) create mode 100644 src/demo_starknet_events/abi/stark_usdt/cairo_abi.json diff --git a/src/demo_starknet_events/abi/stark_usdt/cairo_abi.json b/src/demo_starknet_events/abi/stark_usdt/cairo_abi.json new file mode 100644 index 000000000..718a2577b --- /dev/null +++ b/src/demo_starknet_events/abi/stark_usdt/cairo_abi.json @@ -0,0 +1,1029 @@ +[ + { + "type": "impl", + "name": "MintableToken", + "interface_name": "src::mintable_token_interface::IMintableToken" + }, + { + "type": "struct", + "name": "core::integer::u256", + "members": [ + { + "type": "core::integer::u128", + "name": "low" + }, + { + "type": "core::integer::u128", + "name": "high" + } + ] + }, + { + "type": "interface", + "name": "src::mintable_token_interface::IMintableToken", + "items": [ + { + "name": "permissioned_mint", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + }, + { + "type": "core::integer::u256", + "name": "amount" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + }, + { + "name": "permissioned_burn", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + }, + { + "type": "core::integer::u256", + "name": "amount" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + } + ] + }, + { + "type": "impl", + "name": "MintableTokenCamelImpl", + "interface_name": "src::mintable_token_interface::IMintableTokenCamel" + }, + { + "type": "interface", + "name": "src::mintable_token_interface::IMintableTokenCamel", + "items": [ + { + "name": "permissionedMint", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + }, + { + "type": "core::integer::u256", + "name": "amount" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + }, + { + "name": "permissionedBurn", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + }, + { + "type": "core::integer::u256", + "name": "amount" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + } + ] + }, + { + "type": "impl", + "name": "Replaceable", + "interface_name": "src::replaceability_interface::IReplaceable" + }, + { + "type": "struct", + "name": "core::array::Span::", + "members": [ + { + "type": "@core::array::Array::", + "name": "snapshot" + } + ] + }, + { + "type": "struct", + "name": "src::replaceability_interface::EICData", + "members": [ + { + "type": "core::starknet::class_hash::ClassHash", + "name": "eic_hash" + }, + { + "type": "core::array::Span::", + "name": "eic_init_data" + } + ] + }, + { + "type": "enum", + "name": "core::option::Option::", + "variants": [ + { + "type": "src::replaceability_interface::EICData", + "name": "Some" + }, + { + "type": "()", + "name": "None" + } + ] + }, + { + "type": "enum", + "name": "core::bool", + "variants": [ + { + "type": "()", + "name": "False" + }, + { + "type": "()", + "name": "True" + } + ] + }, + { + "type": "struct", + "name": "src::replaceability_interface::ImplementationData", + "members": [ + { + "type": "core::starknet::class_hash::ClassHash", + "name": "impl_hash" + }, + { + "type": "core::option::Option::", + "name": "eic_data" + }, + { + "type": "core::bool", + "name": "final" + } + ] + }, + { + "type": "interface", + "name": "src::replaceability_interface::IReplaceable", + "items": [ + { + "name": "get_upgrade_delay", + "inputs": [], + "outputs": [ + { + "type": "core::integer::u64" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "get_impl_activation_time", + "inputs": [ + { + "type": "src::replaceability_interface::ImplementationData", + "name": "implementation_data" + } + ], + "outputs": [ + { + "type": "core::integer::u64" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "add_new_implementation", + "inputs": [ + { + "type": "src::replaceability_interface::ImplementationData", + "name": "implementation_data" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + }, + { + "name": "remove_implementation", + "inputs": [ + { + "type": "src::replaceability_interface::ImplementationData", + "name": "implementation_data" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + }, + { + "name": "replace_to", + "inputs": [ + { + "type": "src::replaceability_interface::ImplementationData", + "name": "implementation_data" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + } + ] + }, + { + "type": "impl", + "name": "AccessControlImplExternal", + "interface_name": "src::access_control_interface::IAccessControl" + }, + { + "type": "interface", + "name": "src::access_control_interface::IAccessControl", + "items": [ + { + "name": "has_role", + "inputs": [ + { + "type": "core::felt252", + "name": "role" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "get_role_admin", + "inputs": [ + { + "type": "core::felt252", + "name": "role" + } + ], + "outputs": [ + { + "type": "core::felt252" + } + ], + "state_mutability": "view", + "type": "function" + } + ] + }, + { + "type": "impl", + "name": "RolesImpl", + "interface_name": "src::roles_interface::IMinimalRoles" + }, + { + "type": "interface", + "name": "src::roles_interface::IMinimalRoles", + "items": [ + { + "name": "is_governance_admin", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "is_upgrade_governor", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "register_governance_admin", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + }, + { + "name": "remove_governance_admin", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + }, + { + "name": "register_upgrade_governor", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + }, + { + "name": "remove_upgrade_governor", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + }, + { + "name": "renounce", + "inputs": [ + { + "type": "core::felt252", + "name": "role" + } + ], + "outputs": [], + "state_mutability": "external", + "type": "function" + } + ] + }, + { + "type": "impl", + "name": "ERC20Impl", + "interface_name": "openzeppelin::token::erc20::interface::IERC20" + }, + { + "type": "interface", + "name": "openzeppelin::token::erc20::interface::IERC20", + "items": [ + { + "name": "name", + "inputs": [], + "outputs": [ + { + "type": "core::felt252" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "symbol", + "inputs": [], + "outputs": [ + { + "type": "core::felt252" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "decimals", + "inputs": [], + "outputs": [ + { + "type": "core::integer::u8" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "total_supply", + "inputs": [], + "outputs": [ + { + "type": "core::integer::u256" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "balance_of", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + } + ], + "outputs": [ + { + "type": "core::integer::u256" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "allowance", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "owner" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "spender" + } + ], + "outputs": [ + { + "type": "core::integer::u256" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "transfer", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "recipient" + }, + { + "type": "core::integer::u256", + "name": "amount" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "external", + "type": "function" + }, + { + "name": "transfer_from", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "sender" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "recipient" + }, + { + "type": "core::integer::u256", + "name": "amount" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "external", + "type": "function" + }, + { + "name": "approve", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "spender" + }, + { + "type": "core::integer::u256", + "name": "amount" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "external", + "type": "function" + } + ] + }, + { + "type": "impl", + "name": "ERC20CamelOnlyImpl", + "interface_name": "openzeppelin::token::erc20::interface::IERC20CamelOnly" + }, + { + "type": "interface", + "name": "openzeppelin::token::erc20::interface::IERC20CamelOnly", + "items": [ + { + "name": "totalSupply", + "inputs": [], + "outputs": [ + { + "type": "core::integer::u256" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "balanceOf", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account" + } + ], + "outputs": [ + { + "type": "core::integer::u256" + } + ], + "state_mutability": "view", + "type": "function" + }, + { + "name": "transferFrom", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "sender" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "recipient" + }, + { + "type": "core::integer::u256", + "name": "amount" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "external", + "type": "function" + } + ] + }, + { + "type": "constructor", + "name": "constructor", + "inputs": [ + { + "type": "core::felt252", + "name": "name" + }, + { + "type": "core::felt252", + "name": "symbol" + }, + { + "type": "core::integer::u8", + "name": "decimals" + }, + { + "type": "core::integer::u256", + "name": "initial_supply" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "recipient" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "permitted_minter" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "provisional_governance_admin" + }, + { + "type": "core::integer::u64", + "name": "upgrade_delay" + } + ] + }, + { + "name": "increase_allowance", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "spender" + }, + { + "type": "core::integer::u256", + "name": "added_value" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "external", + "type": "function" + }, + { + "name": "decrease_allowance", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "spender" + }, + { + "type": "core::integer::u256", + "name": "subtracted_value" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "external", + "type": "function" + }, + { + "name": "increaseAllowance", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "spender" + }, + { + "type": "core::integer::u256", + "name": "addedValue" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "external", + "type": "function" + }, + { + "name": "decreaseAllowance", + "inputs": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "spender" + }, + { + "type": "core::integer::u256", + "name": "subtractedValue" + } + ], + "outputs": [ + { + "type": "core::bool" + } + ], + "state_mutability": "external", + "type": "function" + }, + { + "type": "event", + "name": "openzeppelin::token::erc20_v070::erc20::ERC20::Transfer", + "kind": "struct", + "members": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "from", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "to", + "kind": "data" + }, + { + "type": "core::integer::u256", + "name": "value", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "openzeppelin::token::erc20_v070::erc20::ERC20::Approval", + "kind": "struct", + "members": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "owner", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "spender", + "kind": "data" + }, + { + "type": "core::integer::u256", + "name": "value", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::replaceability_interface::ImplementationAdded", + "kind": "struct", + "members": [ + { + "type": "src::replaceability_interface::ImplementationData", + "name": "implementation_data", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::replaceability_interface::ImplementationRemoved", + "kind": "struct", + "members": [ + { + "type": "src::replaceability_interface::ImplementationData", + "name": "implementation_data", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::replaceability_interface::ImplementationReplaced", + "kind": "struct", + "members": [ + { + "type": "src::replaceability_interface::ImplementationData", + "name": "implementation_data", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::replaceability_interface::ImplementationFinalized", + "kind": "struct", + "members": [ + { + "type": "core::starknet::class_hash::ClassHash", + "name": "impl_hash", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::access_control_interface::RoleGranted", + "kind": "struct", + "members": [ + { + "type": "core::felt252", + "name": "role", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "sender", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::access_control_interface::RoleRevoked", + "kind": "struct", + "members": [ + { + "type": "core::felt252", + "name": "role", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "account", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "sender", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::access_control_interface::RoleAdminChanged", + "kind": "struct", + "members": [ + { + "type": "core::felt252", + "name": "role", + "kind": "data" + }, + { + "type": "core::felt252", + "name": "previous_admin_role", + "kind": "data" + }, + { + "type": "core::felt252", + "name": "new_admin_role", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::roles_interface::GovernanceAdminAdded", + "kind": "struct", + "members": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "added_account", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "added_by", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::roles_interface::GovernanceAdminRemoved", + "kind": "struct", + "members": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "removed_account", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "removed_by", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::roles_interface::UpgradeGovernorAdded", + "kind": "struct", + "members": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "added_account", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "added_by", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "src::roles_interface::UpgradeGovernorRemoved", + "kind": "struct", + "members": [ + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "removed_account", + "kind": "data" + }, + { + "type": "core::starknet::contract_address::ContractAddress", + "name": "removed_by", + "kind": "data" + } + ] + }, + { + "type": "event", + "name": "openzeppelin::token::erc20_v070::erc20::ERC20::Event", + "kind": "enum", + "variants": [ + { + "type": "openzeppelin::token::erc20_v070::erc20::ERC20::Transfer", + "name": "Transfer", + "kind": "nested" + }, + { + "type": "openzeppelin::token::erc20_v070::erc20::ERC20::Approval", + "name": "Approval", + "kind": "nested" + }, + { + "type": "src::replaceability_interface::ImplementationAdded", + "name": "ImplementationAdded", + "kind": "nested" + }, + { + "type": "src::replaceability_interface::ImplementationRemoved", + "name": "ImplementationRemoved", + "kind": "nested" + }, + { + "type": "src::replaceability_interface::ImplementationReplaced", + "name": "ImplementationReplaced", + "kind": "nested" + }, + { + "type": "src::replaceability_interface::ImplementationFinalized", + "name": "ImplementationFinalized", + "kind": "nested" + }, + { + "type": "src::access_control_interface::RoleGranted", + "name": "RoleGranted", + "kind": "nested" + }, + { + "type": "src::access_control_interface::RoleRevoked", + "name": "RoleRevoked", + "kind": "nested" + }, + { + "type": "src::access_control_interface::RoleAdminChanged", + "name": "RoleAdminChanged", + "kind": "nested" + }, + { + "type": "src::roles_interface::GovernanceAdminAdded", + "name": "GovernanceAdminAdded", + "kind": "nested" + }, + { + "type": "src::roles_interface::GovernanceAdminRemoved", + "name": "GovernanceAdminRemoved", + "kind": "nested" + }, + { + "type": "src::roles_interface::UpgradeGovernorAdded", + "name": "UpgradeGovernorAdded", + "kind": "nested" + }, + { + "type": "src::roles_interface::UpgradeGovernorRemoved", + "name": "UpgradeGovernorRemoved", + "kind": "nested" + } + ] + } +] \ No newline at end of file From aac72d2ac83a4d1d61774fdd3c8d318f2dd59833 Mon Sep 17 00:00:00 2001 From: baitcode Date: Sun, 2 Feb 2025 23:25:51 +0300 Subject: [PATCH 10/12] review fixes --- src/dipdup/abi/evm.py | 8 ++------ src/dipdup/codegen/starknet.py | 2 +- src/dipdup/datasources/__init__.py | 2 +- src/dipdup/datasources/evm_etherscan.py | 2 +- src/dipdup/datasources/starknet_node.py | 4 ++-- src/dipdup/datasources/substrate_subscan.py | 2 +- 6 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/dipdup/abi/evm.py b/src/dipdup/abi/evm.py index cdc5c2ad4..21f66fc2e 100644 --- a/src/dipdup/abi/evm.py +++ b/src/dipdup/abi/evm.py @@ -96,12 +96,8 @@ def convert_abi(package: DipDupPackage) -> dict[str, EvmAbi]: abi_by_typename: dict[str, EvmAbi] = {} for abi_path in package.evm_abi_paths: - try: - converted_abi = _convert_abi(abi_path) - abi_by_typename[abi_path.parent.stem] = converted_abi - except KeyError as e: - _logger.error('Evm Abi manager can not decod abi file %s', abi_path, exc_info=e) - pass + converted_abi = _convert_abi(abi_path) + abi_by_typename[abi_path.parent.stem] = converted_abi return abi_by_typename diff --git a/src/dipdup/codegen/starknet.py b/src/dipdup/codegen/starknet.py index 2c6dc210d..e92be1a87 100644 --- a/src/dipdup/codegen/starknet.py +++ b/src/dipdup/codegen/starknet.py @@ -71,7 +71,7 @@ async def _fetch_abi(self, index_config: StarknetEventsIndexConfig) -> None: ) touch(abi_path) - abi_path.write_bytes(json_dumps(abi_json['cairo_abi'])) + abi_path.write_bytes(json_dumps(abi_json)) async def generate_schemas(self) -> None: from dipdup.abi.cairo import abi_to_jsonschemas diff --git a/src/dipdup/datasources/__init__.py b/src/dipdup/datasources/__init__.py index d4a6ff69a..9abe6e8b5 100644 --- a/src/dipdup/datasources/__init__.py +++ b/src/dipdup/datasources/__init__.py @@ -58,7 +58,7 @@ async def run(self) -> None: class AbiDatasource(Datasource[DatasourceConfigT], Generic[DatasourceConfigT]): @abstractmethod - async def get_abi(self, address: str) -> dict[str, Any]: ... + async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: ... # FIXME: inconsistent usage diff --git a/src/dipdup/datasources/evm_etherscan.py b/src/dipdup/datasources/evm_etherscan.py index 34fdd3fef..7f4b278c0 100644 --- a/src/dipdup/datasources/evm_etherscan.py +++ b/src/dipdup/datasources/evm_etherscan.py @@ -24,7 +24,7 @@ class EvmEtherscanDatasource(AbiDatasource[EvmEtherscanDatasourceConfig]): async def run(self) -> None: pass - async def get_abi(self, address: str) -> dict[str, Any]: + async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: params = { 'module': 'contract', 'action': 'getabi', diff --git a/src/dipdup/datasources/starknet_node.py b/src/dipdup/datasources/starknet_node.py index ae963157d..1a1f5c8d4 100644 --- a/src/dipdup/datasources/starknet_node.py +++ b/src/dipdup/datasources/starknet_node.py @@ -74,7 +74,7 @@ async def get_events( continuation_token=continuation_token, ) - async def get_abi(self, address: str) -> dict[str, Any]: + async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: from starknet_py.net.client_models import DeprecatedContractClass from starknet_py.net.client_models import SierraContractClass @@ -88,4 +88,4 @@ async def get_abi(self, address: str) -> dict[str, Any]: else: raise NotImplementedError(f'Unknown response class: {class_at_response}') - return {'cairo_abi': parsed_abi} + return parsed_abi or [] diff --git a/src/dipdup/datasources/substrate_subscan.py b/src/dipdup/datasources/substrate_subscan.py index 3344ee9b0..1de9bbb5c 100644 --- a/src/dipdup/datasources/substrate_subscan.py +++ b/src/dipdup/datasources/substrate_subscan.py @@ -7,7 +7,7 @@ class SubstrateSubscanDatasource(AbiDatasource[SubstrateSubscanDatasourceConfig]): # FIXME: not used in codegen - async def get_abi(self, address: str) -> dict[str, Any]: + async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: raise NotImplementedError async def run(self) -> None: From 4f5d3945d76a429b652ad186ddd1830521b001c0 Mon Sep 17 00:00:00 2001 From: baitcode Date: Mon, 3 Feb 2025 02:00:36 +0300 Subject: [PATCH 11/12] Additional fix --- src/dipdup/codegen/evm.py | 66 +++++++++++---------- src/dipdup/codegen/starknet.py | 36 +++-------- src/dipdup/datasources/__init__.py | 41 ++++++++++++- src/dipdup/datasources/starknet_node.py | 4 +- src/dipdup/datasources/substrate_subscan.py | 3 +- 5 files changed, 87 insertions(+), 63 deletions(-) diff --git a/src/dipdup/codegen/evm.py b/src/dipdup/codegen/evm.py index 9afee34e1..9650ccf70 100644 --- a/src/dipdup/codegen/evm.py +++ b/src/dipdup/codegen/evm.py @@ -1,4 +1,6 @@ +from itertools import chain from pathlib import Path +from typing import TYPE_CHECKING from typing import Any from typing import cast @@ -13,13 +15,14 @@ from dipdup.config.evm_transactions import EvmTransactionsHandlerConfig from dipdup.config.evm_transactions import EvmTransactionsIndexConfig from dipdup.datasources import AbiDatasource -from dipdup.exceptions import AbiNotAvailableError from dipdup.exceptions import ConfigurationError -from dipdup.exceptions import DatasourceError from dipdup.utils import json_dumps from dipdup.utils import snake_to_pascal from dipdup.utils import touch +if TYPE_CHECKING: + from collections.abc import Iterable + class EvmCodeGenerator(CodeGenerator): kind = 'evm' @@ -60,42 +63,41 @@ async def generate_handlers(self) -> None: pass async def _fetch_abi(self, index_config: EvmIndexConfigU) -> None: - datasource_configs = tuple(c for c in index_config.datasources if isinstance(c, EvmEtherscanDatasourceConfig)) + contracts_from_event_handlers: list[EvmContractConfig] = [ + handler_config.contract + for handler_config in index_config.handlers + if isinstance(handler_config, EvmEventsHandlerConfig) + ] + contracts_from_transactions: list[EvmContractConfig] = [ + handler_config.typed_contract + for handler_config in index_config.handlers + if ( + isinstance(handler_config, EvmTransactionsHandlerConfig) + and handler_config.typed_contract is not None + ) + ] + contracts: Iterable[EvmContractConfig] = chain(contracts_from_event_handlers, contracts_from_transactions) + + if not contracts: + self._logger.debug('No contract specified. No ABI to fetch.') + return - contract: EvmContractConfig | None = None + datasources: list[AbiDatasource[Any]] = list( + { + datasource_config.name: cast(AbiDatasource[Any], self._datasources[datasource_config.name]) + for datasource_config in index_config.datasources + if isinstance(datasource_config, EvmEtherscanDatasourceConfig) + }.values() + ) - for handler_config in index_config.handlers: - if isinstance(handler_config, EvmEventsHandlerConfig): - contract = handler_config.contract - elif isinstance(handler_config, EvmTransactionsHandlerConfig): - contract = handler_config.typed_contract - - if not contract: - continue + if not datasources: + raise ConfigurationError('No EVM ABI datasources found') + async for contract, abi_json in AbiDatasource.lookup_abi_for(contracts, using=datasources, logger=self._logger): abi_path = self._package.abi / contract.module_name / 'abi.json' + if abi_path.exists(): continue - if not datasource_configs: - raise ConfigurationError('No EVM ABI datasources found') - - address = contract.address or contract.abi - if not address: - raise ConfigurationError(f'`address` or `abi` must be specified for contract `{contract.module_name}`') - - for datasource_config in datasource_configs: - - datasource = cast(AbiDatasource[Any], self._datasources[datasource_config.name]) - try: - abi_json = await datasource.get_abi(address) - break - except DatasourceError as e: - self._logger.warning('Failed to fetch ABI from `%s`: %s', datasource_config.name, e) - else: - raise AbiNotAvailableError( - address=address, - typename=contract.module_name, - ) touch(abi_path) abi_path.write_bytes(json_dumps(abi_json)) diff --git a/src/dipdup/codegen/starknet.py b/src/dipdup/codegen/starknet.py index e92be1a87..12c1c2b52 100644 --- a/src/dipdup/codegen/starknet.py +++ b/src/dipdup/codegen/starknet.py @@ -9,9 +9,7 @@ from dipdup.config.starknet_events import StarknetEventsIndexConfig from dipdup.config.starknet_node import StarknetNodeDatasourceConfig from dipdup.datasources import AbiDatasource -from dipdup.exceptions import AbiNotAvailableError from dipdup.exceptions import ConfigurationError -from dipdup.exceptions import DatasourceError from dipdup.utils import json_dumps from dipdup.utils import snake_to_pascal from dipdup.utils import touch @@ -36,40 +34,24 @@ async def _fetch_abi(self, index_config: StarknetEventsIndexConfig) -> None: self._logger.debug('No contract specified. No ABI to fetch.') return - datasources: list[AbiDatasource[Any]] = [ - cast(AbiDatasource[Any], self._datasources[datasource_config.name]) - for datasource_config in index_config.datasources - if isinstance(datasource_config, StarknetNodeDatasourceConfig) - ] + # deduplicated (by name) Datasource list + datasources: list[AbiDatasource[Any]] = list( + { + datasource_config.name: cast(AbiDatasource[Any], self._datasources[datasource_config.name]) + for datasource_config in index_config.datasources + if isinstance(datasource_config, StarknetNodeDatasourceConfig) + }.values() + ) if not datasources: raise ConfigurationError('No Starknet ABI datasources found') - for contract in contracts: + async for contract, abi_json in AbiDatasource.lookup_abi_for(contracts, using=datasources, logger=self._logger): abi_path = self._package.abi / contract.module_name / 'cairo_abi.json' if abi_path.exists(): continue - abi_json = None - - address = contract.address or contract.abi - if not address: - raise ConfigurationError(f'`address` or `abi` must be specified for contract `{contract.module_name}`') - - for datasource in datasources: - try: - abi_json = await datasource.get_abi(address=address) - break - except DatasourceError as e: - self._logger.warning('Failed to fetch ABI from `%s`: %s', datasource.name, e) - - if abi_json is None: - raise AbiNotAvailableError( - address=address, - typename=contract.module_name, - ) - touch(abi_path) abi_path.write_bytes(json_dumps(abi_json)) diff --git a/src/dipdup/datasources/__init__.py b/src/dipdup/datasources/__init__.py index 9abe6e8b5..f87e1aa92 100644 --- a/src/dipdup/datasources/__init__.py +++ b/src/dipdup/datasources/__init__.py @@ -1,8 +1,11 @@ import asyncio import time from abc import abstractmethod +from collections.abc import AsyncIterator from collections.abc import Awaitable from collections.abc import Callable +from collections.abc import Iterable +from logging import Logger from typing import Any from typing import Generic from typing import TypeVar @@ -15,6 +18,10 @@ from dipdup.config import HttpConfig from dipdup.config import IndexConfig from dipdup.config import ResolvedHttpConfig +from dipdup.config.evm import EvmContractConfig +from dipdup.config.starknet import StarknetContractConfig +from dipdup.exceptions import AbiNotAvailableError +from dipdup.exceptions import ConfigurationError from dipdup.exceptions import DatasourceError from dipdup.exceptions import FrameworkException from dipdup.http import HTTPGateway @@ -29,10 +36,12 @@ from dipdup.utils import FormattedLogger DatasourceConfigT = TypeVar('DatasourceConfigT', bound=DatasourceConfig) +ContractConfigT = TypeVar('ContractConfigT', bound=StarknetContractConfig | EvmContractConfig) EmptyCallback = Callable[[], Awaitable[None]] RollbackCallback = Callable[['IndexDatasource[Any]', MessageType, int, int], Awaitable[None]] +AbiJson = dict[str, Any] | list[Any] class Datasource(HTTPGateway, Generic[DatasourceConfigT]): @@ -57,8 +66,38 @@ async def run(self) -> None: class AbiDatasource(Datasource[DatasourceConfigT], Generic[DatasourceConfigT]): + @abstractmethod - async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: ... + async def get_abi(self, address: str) -> AbiJson: ... + + @staticmethod + async def lookup_abi_for( + contracts: Iterable[ContractConfigT], using: list['AbiDatasource[DatasourceConfigT]'], logger: Logger + ) -> AsyncIterator[tuple[ContractConfigT, AbiJson]]: + """For every contract goes over each datasourse and tries to obtain abi file. + If no ABI exists for any of the contracts - raises error. + """ + for contract in contracts: + abi_json = None + + address = contract.address or contract.abi + if not address: + raise ConfigurationError(f'`address` or `abi` must be specified for contract `{contract.module_name}`') + + for datasource in using: + try: + abi_json = await datasource.get_abi(address=address) + break + except DatasourceError as e: + logger.warning('Failed to fetch ABI from `%s`: %s', datasource.name, e) + + if abi_json is None: + raise AbiNotAvailableError( + address=address, + typename=contract.module_name, + ) + + yield (contract, abi_json) # FIXME: inconsistent usage diff --git a/src/dipdup/datasources/starknet_node.py b/src/dipdup/datasources/starknet_node.py index 1a1f5c8d4..fdc711b86 100644 --- a/src/dipdup/datasources/starknet_node.py +++ b/src/dipdup/datasources/starknet_node.py @@ -1,9 +1,9 @@ import asyncio from typing import TYPE_CHECKING -from typing import Any from dipdup.config import HttpConfig from dipdup.config.starknet_node import StarknetNodeDatasourceConfig +from dipdup.datasources import AbiJson from dipdup.datasources import IndexDatasource from dipdup.datasources._starknetpy import StarknetpyClient @@ -74,7 +74,7 @@ async def get_events( continuation_token=continuation_token, ) - async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: + async def get_abi(self, address: str) -> AbiJson: from starknet_py.net.client_models import DeprecatedContractClass from starknet_py.net.client_models import SierraContractClass diff --git a/src/dipdup/datasources/substrate_subscan.py b/src/dipdup/datasources/substrate_subscan.py index 1de9bbb5c..ba453454a 100644 --- a/src/dipdup/datasources/substrate_subscan.py +++ b/src/dipdup/datasources/substrate_subscan.py @@ -3,11 +3,12 @@ from dipdup.config.substrate_subscan import SubstrateSubscanDatasourceConfig from dipdup.datasources import AbiDatasource +from dipdup.datasources import AbiJson class SubstrateSubscanDatasource(AbiDatasource[SubstrateSubscanDatasourceConfig]): # FIXME: not used in codegen - async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: + async def get_abi(self, address: str) -> AbiJson: raise NotImplementedError async def run(self) -> None: From 546c4337d0d4b160f56c3706f6999fa92d502233 Mon Sep 17 00:00:00 2001 From: baitcode Date: Mon, 3 Feb 2025 02:57:55 +0300 Subject: [PATCH 12/12] comment --- src/dipdup/codegen/evm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dipdup/codegen/evm.py b/src/dipdup/codegen/evm.py index 9650ccf70..09a428924 100644 --- a/src/dipdup/codegen/evm.py +++ b/src/dipdup/codegen/evm.py @@ -82,6 +82,7 @@ async def _fetch_abi(self, index_config: EvmIndexConfigU) -> None: self._logger.debug('No contract specified. No ABI to fetch.') return + # deduplicated (by name) Datasource list datasources: list[AbiDatasource[Any]] = list( { datasource_config.name: cast(AbiDatasource[Any], self._datasources[datasource_config.name])