fix(l1): improve witness generation spec conformance (#6877)

**Motivation**

Currently we are failing several [witness generation
tests](https://eth-act.github.io/eest-execution-witness-dashboard/#/group/execution-witness).

**Description**

Fixes the ordering by sorting, and the over-inclusion by:
- Disabling both BAL and speculative warming when an execution witness
is requested
- While disabling BAL warming isn't strictly needed, it's the simpler
solution. There are several edge-cases around accounts created within
the block that have to be handled carefully.
- Fixing over-accessing in OOG scenarios (call that runs out of gas
right after finding out the target is a eip-7702 delegation, or paying
the cold access cost, etc)

Also adds a test runner for the execution witness tests. You can run the
tests with:

```
cd tooling/ef_tests/blockchain && make test-stateless
cd tooling/ef_tests/engine && make test
```

There are also hive
[tests](https://github.com/eth-act/eest-execution-witness-dashboard)
which take significantly longer to execute.

Author: iovoid

crates/blockchain/blockchain.rs

@@ -551,32 +551,27 @@ impl Blockchain {
         vm.db.store = caching_store.clone();
 
         let cancelled = AtomicBool::new(false);
-        let bal_parallel_exec_enabled = self.options.bal_parallel_exec_enabled;
+        // Witness collection also forces sequential execution: parallel lanes
+        // re-read in-block-created state (e.g. a code deployed by an earlier
+        // tx) from the logged store, while sequential execution serves it from
+        // VM caches — recording accesses the canonical execution never makes.
+        let bal_parallel_exec_enabled = self.options.bal_parallel_exec_enabled && !collect_witness;
 
         // Synthesize BAL updates pre-scope so the merkleizer thread can start
         // trie work immediately, in parallel with execution.
         // `--no-bal-parallel-trie` opts out: leave `optimistic_updates = None` so
         // the merkleizer takes the streaming branch (fed by the EVM-side
         // `bal_to_account_updates` send over the channel below).
+        // Witness collection forces the streaming branch too: the sequential
+        // executor (see `bal_parallel_exec_enabled` below) streams per-tx
+        // updates over the channel, which only the streaming merkleizer
+        // consumes — the synthesized path would leave the receiver dropped.
         let optimistic_updates: Option<FxHashMap<Address, BalSynthesisItem>> =
-            if self.options.bal_parallel_trie_enabled {
+            if self.options.bal_parallel_trie_enabled && !collect_witness {
                 bal.map(synthesize_bal_updates)
             } else {
                 None
             };
-        let optimistic_witness: Option<Vec<AccountUpdate>> = if collect_witness {
-            optimistic_updates.as_ref().map(|m| {
-                m.iter()
-                    .map(|(addr, item)| AccountUpdate {
-                        address: *addr,
-                        added_storage: item.added_storage.clone(),
-                        ..Default::default()
-                    })
-                    .collect()
-            })
-        } else {
-            None
-        };
 
         // Synchronously warm all BAL storage slots before the executor thread starts.
         //
@@ -607,8 +602,14 @@ impl Blockchain {
         // Gated by `--no-bal-prefetch`: when the operator disables BAL-driven
         // prefetching, skip the synchronous storage warm too. The warmer thread
         // below already honors the same toggle.
+        // Witness collection records every read that reaches the store-backed
+        // logger beneath the shared cache. The warmer's speculative reads would
+        // be recorded as state accesses the canonical execution never makes,
+        // polluting the witness (e.g. `engine_newPayloadWithWitnessV5`), so
+        // warming is skipped entirely when a witness is being collected.
         #[cfg(all(feature = "rayon", not(feature = "eip-8025")))]
         if self.options.bal_prefetch_enabled
+            && !collect_witness
             && let Some(bal) = bal
         {
             let slots = LEVM::bal_storage_slots(bal);
@@ -625,53 +626,59 @@ impl Blockchain {
                 #[cfg(all(feature = "rayon", not(feature = "eip-8025")))]
                 let bal_prefetch_enabled = self.options.bal_prefetch_enabled;
                 #[cfg(all(feature = "rayon", not(feature = "eip-8025")))]
-                let warm_handle = std::thread::Builder::new()
-                    .name("block_executor_warmer".to_string())
-                    .spawn_scoped(s, move || {
-                        // Warming uses the same caching store, sharing cached state with execution.
-                        // Precompile cache lives inside CachingDatabase, shared automatically.
-                        let start = Instant::now();
-                        if let Some(bal) = bal {
-                            if bal_prefetch_enabled {
-                                // Amsterdam+: BAL-based precise prefetching (no tx re-execution).
-                                if let Err(e) =
-                                    LEVM::warm_block_from_bal(bal, caching_store, cancelled_ref)
-                                {
-                                    debug!("BAL warming failed (non-fatal): {e}");
-                                }
-                            } else if !bal_parallel_exec_enabled {
-                                // --no-bal-prefetch combined with --no-bal-parallel-exec:
-                                // mirror the pre-Amsterdam setup where a parallel speculative
-                                // warmer races ahead of the serial executor. With parallel
-                                // exec still on, we skip warming instead — two parallel passes
-                                // over the same txs would just fight for cores.
-                                if let Err(e) = LEVM::warm_block(
-                                    block,
-                                    caching_store,
-                                    vm_type,
-                                    &NativeCrypto,
-                                    cancelled_ref,
-                                ) {
-                                    debug!("Block warming failed (non-fatal): {e}");
+                let warm_handle = (!collect_witness)
+                    .then(|| {
+                        std::thread::Builder::new()
+                            .name("block_executor_warmer".to_string())
+                            .spawn_scoped(s, move || {
+                                // Warming uses the same caching store, sharing cached state with execution.
+                                // Precompile cache lives inside CachingDatabase, shared automatically.
+                                let start = Instant::now();
+                                if let Some(bal) = bal {
+                                    if bal_prefetch_enabled {
+                                        // Amsterdam+: BAL-based precise prefetching (no tx re-execution).
+                                        if let Err(e) = LEVM::warm_block_from_bal(
+                                            bal,
+                                            caching_store,
+                                            cancelled_ref,
+                                        ) {
+                                            debug!("BAL warming failed (non-fatal): {e}");
+                                        }
+                                    } else if !bal_parallel_exec_enabled {
+                                        // --no-bal-prefetch combined with --no-bal-parallel-exec:
+                                        // mirror the pre-Amsterdam setup where a parallel speculative
+                                        // warmer races ahead of the serial executor. With parallel
+                                        // exec still on, we skip warming instead — two parallel passes
+                                        // over the same txs would just fight for cores.
+                                        if let Err(e) = LEVM::warm_block(
+                                            block,
+                                            caching_store,
+                                            vm_type,
+                                            &NativeCrypto,
+                                            cancelled_ref,
+                                        ) {
+                                            debug!("Block warming failed (non-fatal): {e}");
+                                        }
+                                    }
+                                } else {
+                                    // Pre-Amsterdam / P2P sync: speculative tx re-execution
+                                    if let Err(e) = LEVM::warm_block(
+                                        block,
+                                        caching_store,
+                                        vm_type,
+                                        &NativeCrypto,
+                                        cancelled_ref,
+                                    ) {
+                                        debug!("Block warming failed (non-fatal): {e}");
+                                    }
                                 }
-                            }
-                        } else {
-                            // Pre-Amsterdam / P2P sync: speculative tx re-execution
-                            if let Err(e) = LEVM::warm_block(
-                                block,
-                                caching_store,
-                                vm_type,
-                                &NativeCrypto,
-                                cancelled_ref,
-                            ) {
-                                debug!("Block warming failed (non-fatal): {e}");
-                            }
-                        }
-                        start.elapsed()
+                                start.elapsed()
+                            })
+                            .map_err(|e| {
+                                ChainError::Custom(format!("Failed to spawn warmer thread: {e}"))
+                            })
                     })
-                    .map_err(|e| {
-                        ChainError::Custom(format!("Failed to spawn warmer thread: {e}"))
-                    })?;
+                    .transpose()?;
                 let max_queue_length_ref = &mut max_queue_length;
                 // Channel is needed whenever the merkleizer takes the streaming
                 // branch OR LEVM falls into the sequential path:
@@ -817,9 +824,13 @@ impl Blockchain {
                 });
                 #[cfg(all(feature = "rayon", not(feature = "eip-8025")))]
                 let warmer_duration = warm_handle
-                    .join()
-                    .inspect_err(|e| warn!("Warming thread error: {e:?}"))
-                    .ok()
+                    .map(|handle| {
+                        handle
+                            .join()
+                            .inspect_err(|e| warn!("Warming thread error: {e:?}"))
+                            .ok()
+                            .unwrap_or(Duration::ZERO)
+                    })
                     .unwrap_or(Duration::ZERO);
                 #[cfg(any(not(feature = "rayon"), feature = "eip-8025"))]
                 let warmer_duration = Duration::ZERO;
@@ -829,8 +840,10 @@ impl Blockchain {
             merkleization_result?;
         let (execution_result, produced_bal, exec_end_instant) = execution_result?;
 
-        // Synthesized witness wins when BAL is present; streaming witness wins otherwise.
-        let accumulated_updates = optimistic_witness.or(streaming_witness);
+        // Witness collection forces the streaming merkleizer (synthesized
+        // updates are disabled above), so the streaming witness is the only
+        // possible source of accumulated updates.
+        let accumulated_updates = streaming_witness;
 
         let exec_merkle_end_instant = Instant::now();
 

crates/common/types/block_execution_witness.rs

@@ -13,6 +13,7 @@ use crate::{
 use ethereum_types::{Address, H256, U256};
 use ethrex_crypto::{Crypto, NativeCrypto};
 use ethrex_rlp::error::RLPDecodeError;
+use ethrex_rlp::structs::{Decoder, Encoder};
 use ethrex_rlp::{decode::RLPDecode, encode::RLPEncode};
 use ethrex_trie::{EMPTY_TRIE_HASH, Nibbles, Node, NodeRef, Trie, TrieError};
 use rkyv::with::{Identity, MapKV};
@@ -124,19 +125,80 @@ impl TryFrom<ExecutionWitness> for RpcExecutionWitness {
         for node in value.storage_trie_roots.values() {
             node.encode_subtrie(&mut nodes)?;
         }
+        // Canonical witness ordering (EIP-8025, geth `ExtWitness`): state nodes
+        // and codes sorted lexicographically and deduplicated (identical
+        // storage subtries would otherwise emit identical nodes twice); headers
+        // ascending by block number and deduplicated (the same ancestor header
+        // can be referenced by more than one block).
+        nodes.sort();
+        nodes.dedup();
+        let mut codes = value.codes;
+        codes.sort();
+        codes.dedup();
+        let mut headers = value.block_headers_bytes;
+        headers.sort_by_cached_key(|bytes| {
+            // Undecodable headers sort last; consumers reject them on decode anyway.
+            BlockHeader::decode(bytes)
+                .map(|header| header.number)
+                .unwrap_or(u64::MAX)
+        });
+        // Identical headers share a block number, so they are now adjacent.
+        headers.dedup();
         Ok(Self {
             state: nodes.into_iter().map(Bytes::from).collect(),
             keys: Vec::new(),
-            codes: value.codes.into_iter().map(Bytes::from).collect(),
-            headers: value
-                .block_headers_bytes
-                .into_iter()
-                .map(Bytes::from)
-                .collect(),
+            codes: codes.into_iter().map(Bytes::from).collect(),
+            headers: headers.into_iter().map(Bytes::from).collect(),
         })
     }
 }
 
+/// Geth's `ExtWitness` wire shape, returned by
+/// `engine_newPayloadWithWitness*`: an RLP list of
+/// `(headers, codes, state, keys)`, with headers ascending by block number
+/// and code/state byte lists sorted lexicographically. Geth currently emits
+/// empty `keys`, but the trailing field is part of the RLP shape.
+/// Reference:
+/// https://github.com/ethereum/go-ethereum/blob/4daaaadfc4706b0a49d4dfde3559de7be968c28a/core/stateless/encoding.go#L30-L52
+#[derive(Debug, Clone)]
+pub struct ExtWitness {
+    pub headers: Vec<BlockHeader>,
+    pub codes: Vec<Bytes>,
+    pub state: Vec<Bytes>,
+    pub keys: Vec<Bytes>,
+}
+
+impl RLPEncode for ExtWitness {
+    fn encode(&self, buf: &mut dyn bytes::BufMut) {
+        Encoder::new(buf)
+            .encode_field(&self.headers)
+            .encode_field(&self.codes)
+            .encode_field(&self.state)
+            .encode_field(&self.keys)
+            .finish();
+    }
+}
+
+impl RLPDecode for ExtWitness {
+    fn decode_unfinished(rlp: &[u8]) -> Result<(Self, &[u8]), RLPDecodeError> {
+        let decoder = Decoder::new(rlp)?;
+        let (headers, decoder) = decoder.decode_field("headers")?;
+        let (codes, decoder) = decoder.decode_field("codes")?;
+        let (state, decoder) = decoder.decode_field("state")?;
+        let (keys, decoder) = decoder.decode_field("keys")?;
+        let remaining = decoder.finish()?;
+        Ok((
+            ExtWitness {
+                headers,
+                codes,
+                state,
+                keys,
+            },
+            remaining,
+        ))
+    }
+}
+
 impl RpcExecutionWitness {
     /// Convert an RPC execution witness into the internal [`ExecutionWitness`]
     /// format by rebuilding trie structures from the flat node list.

crates/networking/rpc/engine/payload.rs

@@ -2,13 +2,15 @@ use bytes::Bytes;
 use ethrex_blockchain::error::ChainError;
 use ethrex_blockchain::payload::PayloadBuildResult;
 use ethrex_common::types::block_access_list::BlockAccessList;
-use ethrex_common::types::block_execution_witness::{ExecutionWitness, RpcExecutionWitness};
+use ethrex_common::types::block_execution_witness::{
+    ExecutionWitness, ExtWitness, RpcExecutionWitness,
+};
 use ethrex_common::types::payload::PayloadBundle;
 use ethrex_common::types::requests::{EncodedRequests, compute_requests_hash};
 use ethrex_common::types::{Block, BlockBody, BlockHash, BlockHeader, BlockNumber, Fork};
 use ethrex_common::{H256, U256};
 use ethrex_p2p::sync::SyncMode;
-use ethrex_rlp::{decode::RLPDecode, error::RLPDecodeError, structs::Encoder};
+use ethrex_rlp::{decode::RLPDecode, encode::RLPEncode, error::RLPDecodeError};
 use serde_json::Value;
 use tokio::sync::oneshot;
 use tracing::{debug, error, info, warn};
@@ -1342,13 +1344,9 @@ fn encode_witness_for_engine_rpc(witness: ExecutionWitness) -> Result<Bytes, Rpc
 /// Encodes the witness in geth's opaque `engine_newPayloadWithWitness*` shape.
 ///
 /// Format: geth returns `rlp.EncodeToBytes(proofs)` from `newPayload`, and
-/// `stateless.Witness::EncodeRLP` delegates to `ExtWitness`.
-/// `ExtWitness` is an RLP list of `(headers, codes, state, keys)`, with
-/// headers ordered by block number and code/state byte lists sorted
-/// lexicographically. Geth currently emits empty keys, but the trailing field
-/// is part of the RLP shape.
-/// Reference:
-/// https://github.com/ethereum/go-ethereum/blob/4daaaadfc4706b0a49d4dfde3559de7be968c28a/core/stateless/encoding.go#L30-L52
+/// `stateless.Witness::EncodeRLP` delegates to [`ExtWitness`] — see its docs
+/// for the shape and geth references.
+/// Additional references:
 /// https://github.com/ethereum/go-ethereum/blob/4daaaadfc4706b0a49d4dfde3559de7be968c28a/core/stateless/encoding.go#L92-L98
 /// https://github.com/ethereum/go-ethereum/blob/4daaaadfc4706b0a49d4dfde3559de7be968c28a/eth/catalyst/api.go#L915-L920
 fn encode_rpc_witness_for_engine_rpc(rpc_witness: RpcExecutionWitness) -> Result<Bytes, RpcErr> {
@@ -1365,14 +1363,13 @@ fn encode_rpc_witness_for_engine_rpc(rpc_witness: RpcExecutionWitness) -> Result
     state.sort_by(|a, b| a.as_ref().cmp(b.as_ref()));
     let mut keys = rpc_witness.keys;
     keys.sort_by(|a, b| a.as_ref().cmp(b.as_ref()));
-    let mut encoded = Vec::new();
-    Encoder::new(&mut encoded)
-        .encode_field(&headers)
-        .encode_field(&codes)
-        .encode_field(&state)
-        .encode_field(&keys)
-        .finish();
-    Ok(Bytes::from(encoded))
+    let ext_witness = ExtWitness {
+        headers,
+        codes,
+        state,
+        keys,
+    };
+    Ok(Bytes::from(ext_witness.encode_to_vec()))
 }
 
 fn parse_get_payload_request(params: &Option<Vec<Value>>) -> Result<u64, RpcErr> {

crates/vm/backends/levm/mod.rs

@@ -2690,10 +2690,11 @@ pub fn generic_system_contract_levm(
         ..Default::default()
     };
 
-    // Invariant relied upon below: with a zero gas price a system call charges no
-    // gas to the SYSTEM_ADDRESS sender and pays no fee to the coinbase, so the only
-    // state change left to undo afterwards is the sender's nonce bump. If this ever
-    // becomes non-zero, the post-call cleanup must be revisited.
+    // Invariant: with a zero gas price (and `is_system_call` making the hook
+    // skip the sender path entirely) a system call leaves no SYSTEM_ADDRESS
+    // state behind — no nonce bump, no balance change, not even a read. If
+    // this ever becomes non-zero, the hook's system-call branches must be
+    // revisited.
     debug_assert!(
         env.gas_price.is_zero() && env.base_fee_per_gas.is_zero(),
         "system calls must run with a zero gas price"
@@ -2734,15 +2735,7 @@ pub fn generic_system_contract_levm(
         recorder.exit_system_call();
     }
 
-    let report = result?;
-
-    // Undo the sender nonce bump: it's the only state change a system call leaves
-    // behind given that the gas price is set to zero.
-    if let Some(account) = db.current_accounts_state.get_mut(&system_address) {
-        account.info.nonce = account.info.nonce.saturating_sub(1);
-    }
-
-    Ok(report)
+    result
 }
 
 #[allow(unreachable_code)]