From 946ccc337ebcc4edc6774d8561bc7c78d0241d91 Mon Sep 17 00:00:00 2001 From: kdeme <7857583+kdeme@users.noreply.github.com> Date: Wed, 25 Jun 2025 18:57:00 +0200 Subject: [PATCH 1/5] Portal: Implementation of finalized history network --- .../content/content_keys.nim | 92 ++++++++ .../content/content_values.nim | 14 ++ .../finalized_history_content.nim | 12 ++ .../finalized_history_network.nim | 204 ++++++++++++++++++ .../finalized_history_validation.nim | 59 +++++ 5 files changed, 381 insertions(+) create mode 100644 portal/network/finalized_history/content/content_keys.nim create mode 100644 portal/network/finalized_history/content/content_values.nim create mode 100644 portal/network/finalized_history/finalized_history_content.nim create mode 100644 portal/network/finalized_history/finalized_history_network.nim create mode 100644 portal/network/finalized_history/finalized_history_validation.nim diff --git a/portal/network/finalized_history/content/content_keys.nim b/portal/network/finalized_history/content/content_keys.nim new file mode 100644 index 000000000..708138d39 --- /dev/null +++ b/portal/network/finalized_history/content/content_keys.nim @@ -0,0 +1,92 @@ +# Nimbus +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + nimcrypto/[sha2, hash], + results, + stint, + ssz_serialization, + ../../../common/common_types + +export ssz_serialization, common_types, results#, hash + +type + ContentType* = enum + # Note: Need to add this unused value as a case object with an enum without + # a 0 valueis not allowed: "low(contentType) must be 0 for discriminant". + # For prefix values that are in the enum gap, the deserialization will fail + # at runtime as is wanted. + # In the future it might be possible that this will fail at compile time for + # the SSZ Union type, but currently it is allowed in the implementation, and + # the SSZ spec is not explicit about disallowing this. + unused = 0x00 + blockBody = 0x09 + receipts = 0x0A + + BlockNumberKey* = object + blockNumber*: uint64 + + ContentKey* = object + case contentType*: ContentType + of unused: + discard + of blockBody: + blockBodyKey*: BlockNumberKey + of receipts: + receiptsKey*: BlockNumberKey + +func blockBodyContentKey*(blockNumber: uint64): ContentKey = + ContentKey(contentType: blockBody, blockBodyKey: BlockNumberKey(blockNumber: blockNumber)) + +func receiptsContentKey*(blockNumber: uint64): ContentKey = + ContentKey(contentType: receipts, receiptsKey: BlockNumberKey(blockNumber: blockNumber)) + +proc readSszBytes*(data: openArray[byte], val: var ContentKey) {.raises: [SszError].} = + mixin readSszValue + if data.len() > 0 and data[0] == ord(unused): + raise newException(MalformedSszError, "SSZ selector is unused value") + + readSszValue(data, val) + +func encode*(contentKey: ContentKey): ContentKeyByteList = + doAssert(contentKey.contentType != unused) + ContentKeyByteList.init(SSZ.encode(contentKey)) + +func decode*(contentKey: ContentKeyByteList): Opt[ContentKey] = + try: + Opt.some(SSZ.decode(contentKey.asSeq(), ContentKey)) + except SerializationError: + return Opt.none(ContentKey) + +# TODO: change to correct content id derivation +func toContentId*(contentKey: ContentKeyByteList): ContentId = + # TODO: Should we try to parse the content key here for invalid ones? + let idHash = sha2.sha256.digest(contentKey.asSeq()) + readUintBE[256](idHash.data) + +func toContentId*(contentKey: ContentKey): ContentId = + toContentId(encode(contentKey)) + +func `$`*(x: BlockNumberKey): string = + "block_number: " & $x.blockNumber + +func `$`*(x: ContentKey): string = + var res = "(type: " & $x.contentType & ", " + + case x.contentType + of unused: + raiseAssert "ContentKey may not have unused value as content type" + of blockBody: + res.add($x.blockBodyKey) + of receipts: + res.add($x.receiptsKey) + + res.add(")") + + res diff --git a/portal/network/finalized_history/content/content_values.nim b/portal/network/finalized_history/content/content_values.nim new file mode 100644 index 000000000..26d21ba0d --- /dev/null +++ b/portal/network/finalized_history/content/content_values.nim @@ -0,0 +1,14 @@ +# Nimbus +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import eth/common/blocks_rlp, eth/common/receipts_rlp + +export blocks_rlp, receipts_rlp + +type Receipts* = seq[Receipt] diff --git a/portal/network/finalized_history/finalized_history_content.nim b/portal/network/finalized_history/finalized_history_content.nim new file mode 100644 index 000000000..461102cca --- /dev/null +++ b/portal/network/finalized_history/finalized_history_content.nim @@ -0,0 +1,12 @@ +# Nimbus +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import ./content/content_keys, ./content/content_values + +export content_keys, content_values diff --git a/portal/network/finalized_history/finalized_history_network.nim b/portal/network/finalized_history/finalized_history_network.nim new file mode 100644 index 000000000..906f32582 --- /dev/null +++ b/portal/network/finalized_history/finalized_history_network.nim @@ -0,0 +1,204 @@ +# Nimbus +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + results, + chronos, + chronicles, + metrics, + eth/common/headers, + eth/p2p/discoveryv5/[protocol, enr], + ../../common/common_types, + ../../database/content_db, + # ../network_metadata, + ../wire/[portal_protocol, portal_stream, portal_protocol_config, ping_extensions], + "."/[finalized_history_content, finalized_history_validation] + +from eth/common/accounts import EMPTY_ROOT_HASH + +logScope: + topics = "portal_fin_hist" + +const pingExtensionCapabilities = {CapabilitiesType, HistoryRadiusType} + +type + FinalizedHistoryNetwork* = ref object + portalProtocol*: PortalProtocol + contentDB*: ContentDB + contentQueue*: AsyncQueue[(Opt[NodeId], ContentKeysList, seq[seq[byte]])] + # cfg*: RuntimeConfig + processContentLoops: seq[Future[void]] + statusLogLoop: Future[void] + contentRequestRetries: int + contentQueueWorkers: int + +func toContentIdHandler(contentKey: ContentKeyByteList): results.Opt[ContentId] = + ok(toContentId(contentKey)) + +proc new*( + T: type FinalizedHistoryNetwork, + portalNetwork: PortalNetwork, + baseProtocol: protocol.Protocol, + contentDB: ContentDB, + streamManager: StreamManager, + # cfg: RuntimeConfig, + bootstrapRecords: openArray[Record] = [], + portalConfig: PortalProtocolConfig = defaultPortalProtocolConfig, + contentRequestRetries = 1, + contentQueueWorkers = 50, + contentQueueSize = 50, +): T = + let + contentQueue = + newAsyncQueue[(Opt[NodeId], ContentKeysList, seq[seq[byte]])](contentQueueSize) + + stream = streamManager.registerNewStream(contentQueue) + + portalProtocol = PortalProtocol.new( + baseProtocol, + [byte(0x50), 0x00], # TODO: Adapt getProtocolId + toContentIdHandler, + createGetHandler(contentDB), + createStoreHandler(contentDB, portalConfig.radiusConfig), + createContainsHandler(contentDB), + createRadiusHandler(contentDB), + stream, + bootstrapRecords, + config = portalConfig, + pingExtensionCapabilities = pingExtensionCapabilities, + ) + + FinalizedHistoryNetwork( + portalProtocol: portalProtocol, + contentDB: contentDB, + contentQueue: contentQueue, + # cfg: cfg, + contentRequestRetries: contentRequestRetries, + contentQueueWorkers: contentQueueWorkers, + ) + +proc validateContent( + n: FinalizedHistoryNetwork, content: seq[byte], contentKeyBytes: ContentKeyByteList +): Future[Result[void, string]] {.async: (raises: [CancelledError]).} = + # TODO: specs might turn out to just disable offers. Although I think for for getting initial data in the network + # this might be an issue. Unless history expiry gets deployed together with Portal. + let contentKey = finalized_history_content.decode(contentKeyBytes).valueOr: + return err("Error decoding content key") + + case contentKey.contentType + of unused: + raiseAssert("ContentKey contentType: unused") + of blockBody: + let + # TODO: Need to get the header (or just tx root/uncle root/withdrawals root) from the EL client via + # JSON-RPC. + # OR if directly integrated the EL client, we can just pass the header here. + header = Header() + blockBody = decodeRlp(content, BlockBody).valueOr: + return err("Error decoding block body: " & error) + validateBlockBody(blockBody, header).isOkOr: + return err("Failed validating block body: " & error) + + ok() + of receipts: + let + # TODO: Need to get the header (or just tx root/uncle root/withdrawals root) from the EL client via + # JSON-RPC. + # OR if directly integrated the EL client, we can just pass the header here. + header = Header() + receipts = decodeRlp(content, seq[Receipt]).valueOr: + return err("Error decoding receipts: " & error) + validateReceipts(receipts, header.receiptsRoot).isOkOr: + return err("Failed validating receipts: " & error) + + ok() + +proc validateContent( + n: FinalizedHistoryNetwork, + srcNodeId: Opt[NodeId], + contentKeys: ContentKeysList, + contentItems: seq[seq[byte]], +): Future[bool] {.async: (raises: [CancelledError]).} = + # content passed here can have less items then contentKeys, but not more. + for i, contentItem in contentItems: + let contentKey = contentKeys[i] + let res = await n.validateContent(contentItem, contentKey) + if res.isOk(): + let contentId = n.portalProtocol.toContentId(contentKey).valueOr: + warn "Received offered content with invalid content key", srcNodeId, contentKey + return false + + n.portalProtocol.storeContent( + contentKey, contentId, contentItem, cacheOffer = true + ) + + debug "Received offered content validated successfully", srcNodeId, contentKey + else: + if srcNodeId.isSome(): + n.portalProtocol.banNode(srcNodeId.get(), NodeBanDurationOfferFailedValidation) + + debug "Received offered content failed validation", + srcNodeId, contentKey, error = res.error + return false + + return true + +proc contentQueueWorker(n: FinalizedHistoryNetwork) {.async: (raises: []).} = + try: + while true: + let (srcNodeId, contentKeys, contentItems) = await n.contentQueue.popFirst() + + if await n.validateContent(srcNodeId, contentKeys, contentItems): + portal_offer_validation_successful.inc( + labelValues = [$n.portalProtocol.protocolId] + ) + + discard await n.portalProtocol.neighborhoodGossip( + srcNodeId, contentKeys, contentItems + ) + else: + portal_offer_validation_failed.inc(labelValues = [$n.portalProtocol.protocolId]) + except CancelledError: + trace "contentQueueWorker canceled" + +proc statusLogLoop(n: FinalizedHistoryNetwork) {.async: (raises: []).} = + try: + while true: + await sleepAsync(60.seconds) + + info "History network status", + routingTableNodes = n.portalProtocol.routingTable.len() + except CancelledError: + trace "statusLogLoop canceled" + +proc start*(n: FinalizedHistoryNetwork) = + info "Starting Portal finalized chain history network", + protocolId = n.portalProtocol.protocolId + + n.portalProtocol.start() + + for i in 0 ..< n.contentQueueWorkers: + n.processContentLoops.add(contentQueueWorker(n)) + + n.statusLogLoop = statusLogLoop(n) + +proc stop*(n: FinalizedHistoryNetwork) {.async: (raises: []).} = + info "Stopping Portal finalized chain history network" + + var futures: seq[Future[void]] + futures.add(n.portalProtocol.stop()) + + for loop in n.processContentLoops: + futures.add(loop.cancelAndWait()) + if not n.statusLogLoop.isNil: + futures.add(n.statusLogLoop.cancelAndWait()) + await noCancel(allFutures(futures)) + + n.processContentLoops.setLen(0) + n.statusLogLoop = nil diff --git a/portal/network/finalized_history/finalized_history_validation.nim b/portal/network/finalized_history/finalized_history_validation.nim new file mode 100644 index 000000000..3c7c6f4fb --- /dev/null +++ b/portal/network/finalized_history/finalized_history_validation.nim @@ -0,0 +1,59 @@ +# Nimbus +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + eth/trie/ordered_trie, + eth/common/[headers_rlp, blocks_rlp, receipts, hashes], + ./finalized_history_content + +func validateBlockBody*( + body: BlockBody, header: Header +): Result[void, string] = + ## Validate the block body against the txRoot, ommersHash and withdrawalsRoot + ## from the header. + ## TODO: could add block number vs empty ommersHash + existing withdrawalsRoot check + let calculatedOmmersHash = keccak256(rlp.encode(body.uncles)) # TODO: avoid having to re-encode the uncles + if calculatedOmmersHash != header.ommersHash: + return err("Invalid ommers hash: expected " & $header.ommersHash & " - got " & + $calculatedOmmersHash) + + let calculatedTxsRoot = orderedTrieRoot(body.transactions) + if calculatedTxsRoot != header.txRoot: + return err( + "Invalid transactions root: expected " & $header.txRoot & " - got " & + $calculatedTxsRoot + ) + + if header.withdrawalsRoot.isSome() and body.withdrawals.isNone() or + header.withdrawalsRoot.isNone() and body.withdrawals.isSome(): + return err( + "Invalid withdrawals" + ) + + if header.withdrawalsRoot.isSome() and body.withdrawals.isSome(): + let + calculatedWithdrawalsRoot = orderedTrieRoot(body.withdrawals.value()) + headerWithdrawalsRoot = header.withdrawalsRoot.get() + if calculatedWithdrawalsRoot != headerWithdrawalsRoot: + return err( + "Invalid withdrawals root: expected " & $headerWithdrawalsRoot & " - got " & + $calculatedWithdrawalsRoot + ) + + ok() + +func validateReceipts*( + receipts: Receipts, receiptsRoot: Hash32 +): Result[void, string] = + let calculatedReceiptsRoot = orderedTrieRoot(receipts) + if calculatedReceiptsRoot != receiptsRoot: + err("Unexpected receipt root: expected " & $receiptsRoot & + " - got " & $calculatedReceiptsRoot) + else: + ok() From e1371f277b897c63b849b94f86a35b4cd435328d Mon Sep 17 00:00:00 2001 From: kdeme <7857583+kdeme@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:57:56 +0200 Subject: [PATCH 2/5] Add content id derivation + content key tests + nph format --- .../content/content_keys.nim | 58 ++++++-- .../finalized_history_network.nim | 21 ++- .../finalized_history_validation.nim | 29 ++-- portal/tests/all_portal_tests.nim | 1 + .../all_finalized_history_network_tests.nim | 10 ++ .../test_finalized_history_content_keys.nim | 128 ++++++++++++++++++ 6 files changed, 206 insertions(+), 41 deletions(-) create mode 100644 portal/tests/finalized_history_network_tests/all_finalized_history_network_tests.nim create mode 100644 portal/tests/finalized_history_network_tests/test_finalized_history_content_keys.nim diff --git a/portal/network/finalized_history/content/content_keys.nim b/portal/network/finalized_history/content/content_keys.nim index 708138d39..7ac7ff1c0 100644 --- a/portal/network/finalized_history/content/content_keys.nim +++ b/portal/network/finalized_history/content/content_keys.nim @@ -7,14 +7,9 @@ {.push raises: [].} -import - nimcrypto/[sha2, hash], - results, - stint, - ssz_serialization, - ../../../common/common_types +import results, stint, ssz_serialization, ../../../common/common_types -export ssz_serialization, common_types, results#, hash +export ssz_serialization, common_types, results type ContentType* = enum @@ -42,10 +37,14 @@ type receiptsKey*: BlockNumberKey func blockBodyContentKey*(blockNumber: uint64): ContentKey = - ContentKey(contentType: blockBody, blockBodyKey: BlockNumberKey(blockNumber: blockNumber)) + ContentKey( + contentType: blockBody, blockBodyKey: BlockNumberKey(blockNumber: blockNumber) + ) func receiptsContentKey*(blockNumber: uint64): ContentKey = - ContentKey(contentType: receipts, receiptsKey: BlockNumberKey(blockNumber: blockNumber)) + ContentKey( + contentType: receipts, receiptsKey: BlockNumberKey(blockNumber: blockNumber) + ) proc readSszBytes*(data: openArray[byte], val: var ContentKey) {.raises: [SszError].} = mixin readSszValue @@ -64,14 +63,43 @@ func decode*(contentKey: ContentKeyByteList): Opt[ContentKey] = except SerializationError: return Opt.none(ContentKey) -# TODO: change to correct content id derivation -func toContentId*(contentKey: ContentKeyByteList): ContentId = - # TODO: Should we try to parse the content key here for invalid ones? - let idHash = sha2.sha256.digest(contentKey.asSeq()) - readUintBE[256](idHash.data) +func reverseBits(n: uint64, width: int): uint64 = + ## Reverse the lowest `width` bits of `n` + # TODO: can improve + var res: uint64 = 0 + for i in 0 ..< width: + if ((n shr i) and 1) != 0: + res = res or (1'u64 shl (width - 1 - i)) + res + +const + CYCLE_BITS = 16 + OFFSET_BITS = 256 - CYCLE_BITS # 240 + REVERSED_OFFSET_BITS = 64 - CYCLE_BITS # 48 + +func toContentId*(blockNumber: uint64): UInt256 = + ## Returns the content id for a given block number + let + cycleBits = blockNumber mod (1'u64 shl CYCLE_BITS) + offsetBits = blockNumber div (1'u64 shl CYCLE_BITS) + + reversedOffsetBits = reverseBits(offsetBits, REVERSED_OFFSET_BITS) + + (cycleBits.stuint(256) shl OFFSET_BITS) or + (reversedOffsetBits.stuint(256) shl (OFFSET_BITS - REVERSED_OFFSET_BITS)) func toContentId*(contentKey: ContentKey): ContentId = - toContentId(encode(contentKey)) + case contentKey.contentType + of unused: + raiseAssert "ContentKey may not have unused value as content type" + of blockBody: + toContentId(contentKey.blockBodyKey.blockNumber) + of receipts: + toContentId(contentKey.receiptsKey.blockNumber) + +func toContentId*(bytes: ContentKeyByteList): Opt[ContentId] = + let contentKey = ?bytes.decode() + Opt.some(contentKey.toContentId()) func `$`*(x: BlockNumberKey): string = "block_number: " & $x.blockNumber diff --git a/portal/network/finalized_history/finalized_history_network.nim b/portal/network/finalized_history/finalized_history_network.nim index 906f32582..0c0e0ddc2 100644 --- a/portal/network/finalized_history/finalized_history_network.nim +++ b/portal/network/finalized_history/finalized_history_network.nim @@ -27,19 +27,18 @@ logScope: const pingExtensionCapabilities = {CapabilitiesType, HistoryRadiusType} -type - FinalizedHistoryNetwork* = ref object - portalProtocol*: PortalProtocol - contentDB*: ContentDB - contentQueue*: AsyncQueue[(Opt[NodeId], ContentKeysList, seq[seq[byte]])] - # cfg*: RuntimeConfig - processContentLoops: seq[Future[void]] - statusLogLoop: Future[void] - contentRequestRetries: int - contentQueueWorkers: int +type FinalizedHistoryNetwork* = ref object + portalProtocol*: PortalProtocol + contentDB*: ContentDB + contentQueue*: AsyncQueue[(Opt[NodeId], ContentKeysList, seq[seq[byte]])] + # cfg*: RuntimeConfig + processContentLoops: seq[Future[void]] + statusLogLoop: Future[void] + contentRequestRetries: int + contentQueueWorkers: int func toContentIdHandler(contentKey: ContentKeyByteList): results.Opt[ContentId] = - ok(toContentId(contentKey)) + toContentId(contentKey) proc new*( T: type FinalizedHistoryNetwork, diff --git a/portal/network/finalized_history/finalized_history_validation.nim b/portal/network/finalized_history/finalized_history_validation.nim index 3c7c6f4fb..13c567bee 100644 --- a/portal/network/finalized_history/finalized_history_validation.nim +++ b/portal/network/finalized_history/finalized_history_validation.nim @@ -12,16 +12,17 @@ import eth/common/[headers_rlp, blocks_rlp, receipts, hashes], ./finalized_history_content -func validateBlockBody*( - body: BlockBody, header: Header -): Result[void, string] = +func validateBlockBody*(body: BlockBody, header: Header): Result[void, string] = ## Validate the block body against the txRoot, ommersHash and withdrawalsRoot ## from the header. ## TODO: could add block number vs empty ommersHash + existing withdrawalsRoot check - let calculatedOmmersHash = keccak256(rlp.encode(body.uncles)) # TODO: avoid having to re-encode the uncles + let calculatedOmmersHash = keccak256(rlp.encode(body.uncles)) + # TODO: avoid having to re-encode the uncles if calculatedOmmersHash != header.ommersHash: - return err("Invalid ommers hash: expected " & $header.ommersHash & " - got " & - $calculatedOmmersHash) + return err( + "Invalid ommers hash: expected " & $header.ommersHash & " - got " & + $calculatedOmmersHash + ) let calculatedTxsRoot = orderedTrieRoot(body.transactions) if calculatedTxsRoot != header.txRoot: @@ -31,10 +32,8 @@ func validateBlockBody*( ) if header.withdrawalsRoot.isSome() and body.withdrawals.isNone() or - header.withdrawalsRoot.isNone() and body.withdrawals.isSome(): - return err( - "Invalid withdrawals" - ) + header.withdrawalsRoot.isNone() and body.withdrawals.isSome(): + return err("Invalid withdrawals") if header.withdrawalsRoot.isSome() and body.withdrawals.isSome(): let @@ -48,12 +47,12 @@ func validateBlockBody*( ok() -func validateReceipts*( - receipts: Receipts, receiptsRoot: Hash32 -): Result[void, string] = +func validateReceipts*(receipts: Receipts, receiptsRoot: Hash32): Result[void, string] = let calculatedReceiptsRoot = orderedTrieRoot(receipts) if calculatedReceiptsRoot != receiptsRoot: - err("Unexpected receipt root: expected " & $receiptsRoot & - " - got " & $calculatedReceiptsRoot) + err( + "Unexpected receipt root: expected " & $receiptsRoot & " - got " & + $calculatedReceiptsRoot + ) else: ok() diff --git a/portal/tests/all_portal_tests.nim b/portal/tests/all_portal_tests.nim index 58445f933..851c15672 100644 --- a/portal/tests/all_portal_tests.nim +++ b/portal/tests/all_portal_tests.nim @@ -11,6 +11,7 @@ import ./evm/all_evm_tests, ./test_content_db, ./wire_protocol_tests/all_wire_protocol_tests, + ./finalized_history_network_tests/all_finalized_history_network_tests, ./history_network_tests/all_history_network_tests, ./beacon_network_tests/all_beacon_network_tests, ./state_network_tests/all_state_network_tests, diff --git a/portal/tests/finalized_history_network_tests/all_finalized_history_network_tests.nim b/portal/tests/finalized_history_network_tests/all_finalized_history_network_tests.nim new file mode 100644 index 000000000..64c0b5d4c --- /dev/null +++ b/portal/tests/finalized_history_network_tests/all_finalized_history_network_tests.nim @@ -0,0 +1,10 @@ +# Nimbus +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.warning[UnusedImport]: off.} + +import ./test_finalized_history_content_keys diff --git a/portal/tests/finalized_history_network_tests/test_finalized_history_content_keys.nim b/portal/tests/finalized_history_network_tests/test_finalized_history_content_keys.nim new file mode 100644 index 000000000..defcfa620 --- /dev/null +++ b/portal/tests/finalized_history_network_tests/test_finalized_history_content_keys.nim @@ -0,0 +1,128 @@ +# Nimbus +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + unittest2, stew/byteutils, ../../network/finalized_history/finalized_history_content + +suite "Finalized History Content Keys": + test "toContentId": + # Input + const blockNumbers = [ + 1.uint64, + 1000.uint64, + 12_345_678.uint64, + uint64.high(), + uint64.high() - 1, + uint64.high() div 2, + uint64.high() div 16 + 1, + 6148914691236517205'u64, + 12297829382473034410'u64, + 11574427654092267680'u64, + ] + + # Output + const contentIds = [ + "0001000000000000000000000000000000000000000000000000000000000000", + "03e8000000000000000000000000000000000000000000000000000000000000", + "614e3d0000000000000000000000000000000000000000000000000000000000", + "ffffffffffffffff000000000000000000000000000000000000000000000000", + "fffeffffffffffff000000000000000000000000000000000000000000000000", + "fffffffffffffffe000000000000000000000000000000000000000000000000", + "0000000000000008000000000000000000000000000000000000000000000000", + "5555aaaaaaaaaaaa000000000000000000000000000000000000000000000000", + "aaaa555555555555000000000000000000000000000000000000000000000000", + "a0a0050505050505000000000000000000000000000000000000000000000000", + ] + + for i in 0 ..< blockNumbers.len(): + let contentId = toContentId(blockNumbers[i]) + + check contentIds[i] == contentId.dumpHex() + + test "BlockBody": + # Input + const blockNumber = 12_345_678.uint64 + + # Output + const + contentKeyHex = "0x094e61bc0000000000" + contentId = + "44012581390156707874310974263613699127815223388818970640389075388176810377216" + # or + contentIdHexBE = + "614e3d0000000000000000000000000000000000000000000000000000000000" + + let contentKey = blockBodyContentKey(blockNumber) + + let encoded = encode(contentKey) + check encoded.asSeq.to0xHex == contentKeyHex + let decoded = decode(encoded) + check decoded.isSome() + + let contentKeyDecoded = decoded.get() + check: + contentKeyDecoded.contentType == contentKey.contentType + contentKeyDecoded.blockBodyKey == contentKey.blockBodyKey + + toContentId(contentKey) == parse(contentId, StUint[256], 10) + # In stint this does BE hex string + toContentId(contentKey).toHex() == contentIdHexBE + + test "Receipts": + # Input + const blockNumber = 12_345_678.uint64 + + # Output + const + contentKeyHex = "0x0a4e61bc0000000000" + contentId = + "44012581390156707874310974263613699127815223388818970640389075388176810377216" + # or + contentIdHexBE = + "614e3d0000000000000000000000000000000000000000000000000000000000" + + let contentKey = receiptsContentKey(blockNumber) + + let encoded = encode(contentKey) + check encoded.asSeq.to0xHex == contentKeyHex + let decoded = decode(encoded) + check decoded.isSome() + + let contentKeyDecoded = decoded.get() + check: + contentKeyDecoded.contentType == contentKey.contentType + contentKeyDecoded.receiptsKey == contentKey.receiptsKey + + toContentId(contentKey) == parse(contentId, StUint[256], 10) + # In stint this does BE hex string + toContentId(contentKey).toHex() == contentIdHexBE + + test "Invalid prefix - 0 value": + let encoded = ContentKeyByteList.init(@[byte 0x00]) + let decoded = decode(encoded) + + check decoded.isErr() + + test "Invalid prefix - before valid range": + let encoded = ContentKeyByteList.init(@[byte 0x08]) + let decoded = decode(encoded) + + check decoded.isErr() + + test "Invalid prefix - after valid range": + let encoded = ContentKeyByteList.init(@[byte 0x0B]) + let decoded = decode(encoded) + + check decoded.isErr() + + test "Invalid key - empty input": + let encoded = ContentKeyByteList.init(@[]) + let decoded = decode(encoded) + + check decoded.isErr() From aa92086cbef6f82f5bca837b5a70f0b55eb7f208 Mon Sep 17 00:00:00 2001 From: kdeme <7857583+kdeme@users.noreply.github.com> Date: Mon, 30 Jun 2025 17:47:24 +0200 Subject: [PATCH 3/5] Enable finalized history network to portal client --- portal/client/nimbus_portal_client_conf.nim | 5 +++- .../finalized_history_network.nim | 2 +- portal/network/portal_node.nim | 25 +++++++++++++++++++ portal/network/wire/portal_protocol.nim | 4 +++ .../network/wire/portal_protocol_config.nim | 1 + 5 files changed, 35 insertions(+), 2 deletions(-) diff --git a/portal/client/nimbus_portal_client_conf.nim b/portal/client/nimbus_portal_client_conf.nim index ffe7e58c6..323f82075 100644 --- a/portal/client/nimbus_portal_client_conf.nim +++ b/portal/client/nimbus_portal_client_conf.nim @@ -106,7 +106,10 @@ type portalSubnetworks* {. desc: "Select which networks (Portal sub-protocols) to enable", - defaultValue: {PortalSubnetwork.history, PortalSubnetwork.beacon}, + defaultValue: { + PortalSubnetwork.finalizedHistory, PortalSubnetwork.history, + PortalSubnetwork.beacon, + }, name: "portal-subnetworks" .}: set[PortalSubnetwork] diff --git a/portal/network/finalized_history/finalized_history_network.nim b/portal/network/finalized_history/finalized_history_network.nim index 0c0e0ddc2..cb059530d 100644 --- a/portal/network/finalized_history/finalized_history_network.nim +++ b/portal/network/finalized_history/finalized_history_network.nim @@ -61,7 +61,7 @@ proc new*( portalProtocol = PortalProtocol.new( baseProtocol, - [byte(0x50), 0x00], # TODO: Adapt getProtocolId + getProtocolId(portalNetwork, PortalSubnetwork.finalizedHistory), toContentIdHandler, createGetHandler(contentDB), createStoreHandler(contentDB, portalConfig.radiusConfig), diff --git a/portal/network/portal_node.nim b/portal/network/portal_node.nim index 3baa83c32..6b42edc5f 100644 --- a/portal/network/portal_node.nim +++ b/portal/network/portal_node.nim @@ -17,6 +17,7 @@ import ../database/content_db, ./network_metadata, ./wire/[portal_stream, portal_protocol_config], + ./finalized_history/finalized_history_network, ./beacon/[beacon_init_loader, beacon_light_client], ./history/[history_network, history_content], ./state/[state_network, state_content] @@ -42,6 +43,7 @@ type discovery: protocol.Protocol contentDB: ContentDB streamManager: StreamManager + finalizedHistoryNetwork*: Opt[FinalizedHistoryNetwork] beaconNetwork*: Opt[BeaconNetwork] historyNetwork*: Opt[HistoryNetwork] stateNetwork*: Opt[StateNetwork] @@ -107,6 +109,24 @@ proc new*( # Get it from binary file containing SSZ encoded accumulator loadAccumulator() + finalizedHistoryNetwork = + if PortalSubnetwork.finalizedHistory in subnetworks: + Opt.some( + FinalizedHistoryNetwork.new( + network, + discovery, + contentDB, + streamManager, + bootstrapRecords = bootstrapRecords, + portalConfig = config.portalConfig, + contentRequestRetries = config.contentRequestRetries, + contentQueueWorkers = config.contentQueueWorkers, + contentQueueSize = config.contentQueueSize, + ) + ) + else: + Opt.none(FinalizedHistoryNetwork) + beaconNetwork = if PortalSubnetwork.beacon in subnetworks: let @@ -196,6 +216,7 @@ proc new*( discovery: discovery, contentDB: contentDB, streamManager: streamManager, + finalizedHistoryNetwork: finalizedHistoryNetwork, beaconNetwork: beaconNetwork, historyNetwork: historyNetwork, stateNetwork: stateNetwork, @@ -229,6 +250,8 @@ proc start*(n: PortalNode) = n.discovery.start() + if n.finalizedHistoryNetwork.isSome(): + n.finalizedHistoryNetwork.value.start() if n.beaconNetwork.isSome(): n.beaconNetwork.value.start() if n.historyNetwork.isSome(): @@ -246,6 +269,8 @@ proc stop*(n: PortalNode) {.async: (raises: []).} = var futures: seq[Future[void]] + if n.finalizedHistoryNetwork.isSome(): + futures.add(n.finalizedHistoryNetwork.value.stop()) if n.beaconNetwork.isSome(): futures.add(n.beaconNetwork.value.stop()) if n.historyNetwork.isSome(): diff --git a/portal/network/wire/portal_protocol.nim b/portal/network/wire/portal_protocol.nim index dc41a1b2e..ca6e54e34 100644 --- a/portal/network/wire/portal_protocol.nim +++ b/portal/network/wire/portal_protocol.nim @@ -308,6 +308,8 @@ func getProtocolId*( case network of PortalNetwork.none, PortalNetwork.mainnet: case subnetwork + of PortalSubnetwork.finalizedHistory: + [portalPrefix, 0x00] of PortalSubnetwork.state: [portalPrefix, 0x0A] of PortalSubnetwork.history: @@ -322,6 +324,8 @@ func getProtocolId*( [portalPrefix, 0x0F] of PortalNetwork.angelfood: case subnetwork + of PortalSubnetwork.finalizedHistory: + [portalPrefix, 0x40] of PortalSubnetwork.state: [portalPrefix, 0x4A] of PortalSubnetwork.history: diff --git a/portal/network/wire/portal_protocol_config.nim b/portal/network/wire/portal_protocol_config.nim index 50576f1ac..b0e17eedb 100644 --- a/portal/network/wire/portal_protocol_config.nim +++ b/portal/network/wire/portal_protocol_config.nim @@ -17,6 +17,7 @@ type # The Portal sub-protocols PortalSubnetwork* = enum + finalizedHistory state history beacon From 1a3cc5a88f69dd7cad054601e28b08d3312447a8 Mon Sep 17 00:00:00 2001 From: kdeme <7857583+kdeme@users.noreply.github.com> Date: Tue, 1 Jul 2025 21:48:54 +0200 Subject: [PATCH 4/5] Portal finalized history: Add native API and JSON-RPC API --- .../content/content_values.nim | 4 +- .../finalized_history_endpoints.nim | 22 +++ .../finalized_history_network.nim | 58 ++++++- .../finalized_history_validation.nim | 29 +++- .../rpc/rpc_portal_finalized_history_api.nim | 148 ++++++++++++++++++ portal/rpc/rpc_types.nim | 2 +- 6 files changed, 257 insertions(+), 6 deletions(-) create mode 100644 portal/network/finalized_history/finalized_history_endpoints.nim create mode 100644 portal/rpc/rpc_portal_finalized_history_api.nim diff --git a/portal/network/finalized_history/content/content_values.nim b/portal/network/finalized_history/content/content_values.nim index 26d21ba0d..49a3ccb50 100644 --- a/portal/network/finalized_history/content/content_values.nim +++ b/portal/network/finalized_history/content/content_values.nim @@ -11,4 +11,6 @@ import eth/common/blocks_rlp, eth/common/receipts_rlp export blocks_rlp, receipts_rlp -type Receipts* = seq[Receipt] +type + Receipts* = seq[Receipt] + ContentValueType* = BlockBody | Receipts diff --git a/portal/network/finalized_history/finalized_history_endpoints.nim b/portal/network/finalized_history/finalized_history_endpoints.nim new file mode 100644 index 000000000..82f29f37b --- /dev/null +++ b/portal/network/finalized_history/finalized_history_endpoints.nim @@ -0,0 +1,22 @@ +# Nimbus +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import results, chronicles, chronos, ./finalized_history_network + +export results, finalized_history_network + +proc getBlockBody*( + n: FinalizedHistoryNetwork, header: Header +): Future[Opt[BlockBody]] {.async: (raises: [CancelledError], raw: true).} = + n.getContent(blockBodyContentKey(header.number), BlockBody, header) + +proc getReceipts*( + n: FinalizedHistoryNetwork, header: Header +): Future[Opt[BlockBody]] {.async: (raises: [CancelledError], raw: true).} = + n.getContent(blockBodyContentKey(header.number), BlockBody, header) diff --git a/portal/network/finalized_history/finalized_history_network.nim b/portal/network/finalized_history/finalized_history_network.nim index cb059530d..5317b694f 100644 --- a/portal/network/finalized_history/finalized_history_network.nim +++ b/portal/network/finalized_history/finalized_history_network.nim @@ -22,6 +22,8 @@ import from eth/common/accounts import EMPTY_ROOT_HASH +export finalized_history_content, headers + logScope: topics = "portal_fin_hist" @@ -82,6 +84,60 @@ proc new*( contentQueueWorkers: contentQueueWorkers, ) +proc getContent*( + n: FinalizedHistoryNetwork, + contentKey: ContentKey, + V: type ContentValueType, + header: Header, +): Future[Opt[V]] {.async: (raises: [CancelledError]).} = + let contentKeyBytes = encode(contentKey) + + logScope: + contentKeyBytes + + let contentId = contentKeyBytes.toContentId().valueOr: + warn "Received invalid content key", contentKeyBytes + return Opt.none(V) + + # Check first locally + n.portalProtocol.getLocalContent(contentKeyBytes, contentId).isErrOr: + let contentValue = decodeRlp(value(), V).valueOr: + raiseAssert("Unable to decode history local content value") + + debug "Fetched local content value" + return Opt.some(contentValue) + + for i in 0 ..< (1 + n.contentRequestRetries): + let + lookupRes = (await n.portalProtocol.contentLookup(contentKeyBytes, contentId)).valueOr: + warn "Failed fetching content from the network" + return Opt.none(V) + + contentValue = decodeRlp(lookupRes.content, V).valueOr: + warn "Unable to decode content value from content lookup" + continue + + validateContent(contentValue, header).isOkOr: + n.portalProtocol.banNode( + lookupRes.receivedFrom.id, NodeBanDurationContentLookupFailedValidation + ) + warn "Error validating retrieved content", error = error + continue + + debug "Fetched valid content from the network" + n.portalProtocol.storeContent( + contentKeyBytes, contentId, lookupRes.content, cacheContent = true + ) + + asyncSpawn n.portalProtocol.triggerPoke( + lookupRes.nodesInterestedInContent, contentKeyBytes, lookupRes.content + ) + + return Opt.some(contentValue) + + # Content was requested `1 + requestRetries` times and all failed on validation + Opt.none(V) + proc validateContent( n: FinalizedHistoryNetwork, content: seq[byte], contentKeyBytes: ContentKeyByteList ): Future[Result[void, string]] {.async: (raises: [CancelledError]).} = @@ -113,7 +169,7 @@ proc validateContent( header = Header() receipts = decodeRlp(content, seq[Receipt]).valueOr: return err("Error decoding receipts: " & error) - validateReceipts(receipts, header.receiptsRoot).isOkOr: + validateReceipts(receipts, header).isOkOr: return err("Failed validating receipts: " & error) ok() diff --git a/portal/network/finalized_history/finalized_history_validation.nim b/portal/network/finalized_history/finalized_history_validation.nim index 13c567bee..01579d903 100644 --- a/portal/network/finalized_history/finalized_history_validation.nim +++ b/portal/network/finalized_history/finalized_history_validation.nim @@ -8,6 +8,7 @@ {.push raises: [].} import + std/typetraits, eth/trie/ordered_trie, eth/common/[headers_rlp, blocks_rlp, receipts, hashes], ./finalized_history_content @@ -47,12 +48,34 @@ func validateBlockBody*(body: BlockBody, header: Header): Result[void, string] = ok() -func validateReceipts*(receipts: Receipts, receiptsRoot: Hash32): Result[void, string] = +func validateReceipts*(receipts: Receipts, header: Header): Result[void, string] = let calculatedReceiptsRoot = orderedTrieRoot(receipts) - if calculatedReceiptsRoot != receiptsRoot: + if calculatedReceiptsRoot != header.receiptsRoot: err( - "Unexpected receipt root: expected " & $receiptsRoot & " - got " & + "Unexpected receipt root: expected " & $header.receiptsRoot & " - got " & $calculatedReceiptsRoot ) else: ok() + +func validateContent*( + content: BlockBody | Receipts, header: Header +): Result[void, string] = + type T = type(content) + when T is BlockBody: + validateBlockBody(content, header) + elif T is Receipts: + validateReceipts(content, header) + +func validateContent*( + key: ContentKey, contentBytes: seq[byte], header: Header +): Result[void, string] = + case key.contentType + of unused: + raiseAssert("ContentKey contentType: unused") + of blockBody: + let content = ?decodeRlp(contentBytes, BlockBody) + validateBlockBody(content, header) + of receipts: + let content = ?decodeRlp(contentBytes, Receipts) + validateReceipts(content, header) diff --git a/portal/rpc/rpc_portal_finalized_history_api.nim b/portal/rpc/rpc_portal_finalized_history_api.nim new file mode 100644 index 000000000..9e6b7b5ea --- /dev/null +++ b/portal/rpc/rpc_portal_finalized_history_api.nim @@ -0,0 +1,148 @@ +# Nimbus +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + json_rpc/rpcserver, + json_serialization/std/tables, + stew/byteutils, + ../common/common_types, + ../network/wire/portal_protocol, + ../network/finalized_history/finalized_history_content, + ../network/finalized_history/finalized_history_validation, + ./rpc_types + +export tables + +# Portal Finalized History Network JSON-RPC API +# Note: +# - This API is not part of the Portal Network specification yet. +# - Lower level API calls are not implemented as they are typically only used for (Hive) +# testing and it is not clear yet of this will be needed in the future. +# - Added the header parameter so that validation can happen on json-rpc server side, +# but it could also be moved to client side. +# - Could also make a less generic API + +ContentInfo.useDefaultSerializationIn JrpcConv +TraceContentLookupResult.useDefaultSerializationIn JrpcConv +TraceObject.useDefaultSerializationIn JrpcConv +NodeMetadata.useDefaultSerializationIn JrpcConv +TraceResponse.useDefaultSerializationIn JrpcConv + +# TODO: It would be cleaner to use the existing getContent/getBlockBody/getReceipts calls for +# less code duplication + automatic retries, but the specific error messages + extra content +# info would need to be added to the existing calls. +proc installPortalFinalizedHistoryApiHandlers*( + rpcServer: RpcServer, p: PortalProtocol +) = + rpcServer.rpc("portal_finalizedHistoryGetContent") do( + contentKeyBytes: string, headerBytes: string + ) -> ContentInfo: + let + contentKeyByteList = ContentKeyByteList.init(hexToSeqByte(contentKeyBytes)) + contentKey = decode(contentKeyByteList).valueOr: + raise invalidKeyErr() + contentId = toContentId(contentKey) + header = decodeRlp(hexToSeqByte(headerBytes), Header).valueOr: + raise invalidRequest((code: -39005, msg: "Failed to decode header: " & error)) + + p.getLocalContent(contentKeyByteList, contentId).isErrOr: + return ContentInfo(content: value.to0xHex(), utpTransfer: false) + + let contentLookupResult = (await p.contentLookup(contentKeyByteList, contentId)).valueOr: + raise contentNotFoundErr() + + validateContent(contentKey, contentLookupResult.content, header).isOkOr: + p.banNode( + contentLookupResult.receivedFrom.id, + NodeBanDurationContentLookupFailedValidation, + ) + raise invalidValueErr() + + p.storeContent( + contentKeyByteList, contentId, contentLookupResult.content, cacheContent = true + ) + + ContentInfo( + content: contentLookupResult.content.to0xHex(), + utpTransfer: contentLookupResult.utpTransfer, + ) + + rpcServer.rpc("portal_finalizedHistoryTraceGetContent") do( + contentKeyBytes: string, headerBytes: string + ) -> TraceContentLookupResult: + let + contentKeyByteList = ContentKeyByteList.init(hexToSeqByte(contentKeyBytes)) + contentKey = decode(contentKeyByteList).valueOr: + raise invalidKeyErr() + contentId = toContentId(contentKey) + header = decodeRlp(hexToSeqByte(headerBytes), Header).valueOr: + raise invalidRequest((code: -39005, msg: "Failed to decode header: " & error)) + + p.getLocalContent(contentKeyByteList, contentId).isErrOr: + return TraceContentLookupResult( + content: Opt.some(value), + utpTransfer: false, + trace: TraceObject( + origin: p.localNode.id, + targetId: contentId, + receivedFrom: Opt.some(p.localNode.id), + ), + ) + + # TODO: Might want to restructure the lookup result here. Potentially doing + # the json conversion in this module. + let + res = await p.traceContentLookup(contentKeyByteList, contentId) + valueBytes = res.content.valueOr: + let data = Opt.some(JrpcConv.encode(res.trace).JsonString) + raise contentNotFoundErrWithTrace(data) + + validateContent(contentKey, valueBytes, header).isOkOr: + if res.trace.receivedFrom.isSome(): + p.banNode( + res.trace.receivedFrom.get(), NodeBanDurationContentLookupFailedValidation + ) + raise invalidValueErr() + + p.storeContent(contentKeyByteList, contentId, valueBytes, cacheContent = true) + + res + + rpcServer.rpc("portal_finalizedHistoryPutContent") do( + contentKeyBytes: string, contentValueBytes: string + ) -> PutContentResult: + let + contentKeyByteList = ContentKeyByteList.init(hexToSeqByte(contentKeyBytes)) + _ = decode(contentKeyByteList).valueOr: + raise invalidKeyErr() + offerValueBytes = hexToSeqByte(contentValueBytes) + + # Note: Not validating content as this would have a high impact on bridge + # gossip performance. + # As no validation is done here, the content is not stored locally. + # TODO: Add default on validation by optional validation parameter. + gossipMetadata = await p.neighborhoodGossip( + Opt.none(NodeId), + ContentKeysList(@[contentKeyByteList]), + @[offerValueBytes], + enableNodeLookup = true, + ) + + PutContentResult( + storedLocally: false, + peerCount: gossipMetadata.successCount, + acceptMetadata: AcceptMetadata( + acceptedCount: gossipMetadata.acceptedCount, + genericDeclineCount: gossipMetadata.genericDeclineCount, + alreadyStoredCount: gossipMetadata.alreadyStoredCount, + notWithinRadiusCount: gossipMetadata.notWithinRadiusCount, + rateLimitedCount: gossipMetadata.rateLimitedCount, + transferInProgressCount: gossipMetadata.transferInProgressCount, + ), + ) diff --git a/portal/rpc/rpc_types.nim b/portal/rpc/rpc_types.nim index f3f8dd086..b5c94373b 100644 --- a/portal/rpc/rpc_types.nim +++ b/portal/rpc/rpc_types.nim @@ -64,7 +64,7 @@ template payloadTypeRequiredError*(): auto = template userSpecifiedPayloadBlockedByClientError*(): auto = UserSpecifiedPayloadBlockedByClientError.applicationError() -template invalidRequest(error: (int, string)): auto = +template invalidRequest*(error: (int, string)): auto = (ref errors.InvalidRequest)(code: error.code, msg: error.msg) template invalidKeyErr*(): auto = From 2b5cebcee778d7343b71c8479281f96938d44d1d Mon Sep 17 00:00:00 2001 From: kdeme <7857583+kdeme@users.noreply.github.com> Date: Sun, 6 Jul 2025 22:53:17 +0200 Subject: [PATCH 5/5] Adjust content id derivation to differentiate body and receipts --- .../content/content_keys.nim | 9 +++--- .../test_finalized_history_content_keys.nim | 30 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/portal/network/finalized_history/content/content_keys.nim b/portal/network/finalized_history/content/content_keys.nim index 7ac7ff1c0..969e66429 100644 --- a/portal/network/finalized_history/content/content_keys.nim +++ b/portal/network/finalized_history/content/content_keys.nim @@ -77,7 +77,7 @@ const OFFSET_BITS = 256 - CYCLE_BITS # 240 REVERSED_OFFSET_BITS = 64 - CYCLE_BITS # 48 -func toContentId*(blockNumber: uint64): UInt256 = +func toContentId*(blockNumber: uint64, contentType: ContentType): UInt256 = ## Returns the content id for a given block number let cycleBits = blockNumber mod (1'u64 shl CYCLE_BITS) @@ -86,16 +86,17 @@ func toContentId*(blockNumber: uint64): UInt256 = reversedOffsetBits = reverseBits(offsetBits, REVERSED_OFFSET_BITS) (cycleBits.stuint(256) shl OFFSET_BITS) or - (reversedOffsetBits.stuint(256) shl (OFFSET_BITS - REVERSED_OFFSET_BITS)) + (reversedOffsetBits.stuint(256) shl (OFFSET_BITS - REVERSED_OFFSET_BITS)) or + ord(contentType).stuint(256) func toContentId*(contentKey: ContentKey): ContentId = case contentKey.contentType of unused: raiseAssert "ContentKey may not have unused value as content type" of blockBody: - toContentId(contentKey.blockBodyKey.blockNumber) + toContentId(contentKey.blockBodyKey.blockNumber, contentKey.contentType) of receipts: - toContentId(contentKey.receiptsKey.blockNumber) + toContentId(contentKey.receiptsKey.blockNumber, contentKey.contentType) func toContentId*(bytes: ContentKeyByteList): Opt[ContentId] = let contentKey = ?bytes.decode() diff --git a/portal/tests/finalized_history_network_tests/test_finalized_history_content_keys.nim b/portal/tests/finalized_history_network_tests/test_finalized_history_content_keys.nim index defcfa620..df79016a0 100644 --- a/portal/tests/finalized_history_network_tests/test_finalized_history_content_keys.nim +++ b/portal/tests/finalized_history_network_tests/test_finalized_history_content_keys.nim @@ -28,20 +28,20 @@ suite "Finalized History Content Keys": # Output const contentIds = [ - "0001000000000000000000000000000000000000000000000000000000000000", - "03e8000000000000000000000000000000000000000000000000000000000000", - "614e3d0000000000000000000000000000000000000000000000000000000000", - "ffffffffffffffff000000000000000000000000000000000000000000000000", - "fffeffffffffffff000000000000000000000000000000000000000000000000", - "fffffffffffffffe000000000000000000000000000000000000000000000000", - "0000000000000008000000000000000000000000000000000000000000000000", - "5555aaaaaaaaaaaa000000000000000000000000000000000000000000000000", - "aaaa555555555555000000000000000000000000000000000000000000000000", - "a0a0050505050505000000000000000000000000000000000000000000000000", + "0001000000000000000000000000000000000000000000000000000000000009", + "03e8000000000000000000000000000000000000000000000000000000000009", + "614e3d0000000000000000000000000000000000000000000000000000000009", + "ffffffffffffffff000000000000000000000000000000000000000000000009", + "fffeffffffffffff000000000000000000000000000000000000000000000009", + "fffffffffffffffe000000000000000000000000000000000000000000000009", + "0000000000000008000000000000000000000000000000000000000000000009", + "5555aaaaaaaaaaaa000000000000000000000000000000000000000000000009", + "aaaa555555555555000000000000000000000000000000000000000000000009", + "a0a0050505050505000000000000000000000000000000000000000000000009", ] for i in 0 ..< blockNumbers.len(): - let contentId = toContentId(blockNumbers[i]) + let contentId = toContentId(blockNumbers[i], ContentType.blockBody) check contentIds[i] == contentId.dumpHex() @@ -53,10 +53,10 @@ suite "Finalized History Content Keys": const contentKeyHex = "0x094e61bc0000000000" contentId = - "44012581390156707874310974263613699127815223388818970640389075388176810377216" + "44012581390156707874310974263613699127815223388818970640389075388176810377225" # or contentIdHexBE = - "614e3d0000000000000000000000000000000000000000000000000000000000" + "614e3d0000000000000000000000000000000000000000000000000000000009" let contentKey = blockBodyContentKey(blockNumber) @@ -82,10 +82,10 @@ suite "Finalized History Content Keys": const contentKeyHex = "0x0a4e61bc0000000000" contentId = - "44012581390156707874310974263613699127815223388818970640389075388176810377216" + "44012581390156707874310974263613699127815223388818970640389075388176810377226" # or contentIdHexBE = - "614e3d0000000000000000000000000000000000000000000000000000000000" + "614e3d000000000000000000000000000000000000000000000000000000000a" let contentKey = receiptsContentKey(blockNumber)