all: implement EIP-compliant verkle trees

verkle: Implement Trie, NodeIterator and Database ifs

Fix crash in TestDump

Fix TestDump

Fix TrieCopy

remove unnecessary traces

fix: Error() returned errIteratorEnd in verkle node iterator

rewrite the iterator and change the signature of OpenStorageTrie

add the adapter to reuse the account trie for storage

don't try to deserialize a storage leaf into an account

Fix statedb unit tests (#14)

* debug code

* Fix more unit tests

* remove traces

* Go back to the full range

One tree to rule them all

remove updateRoot, there is no root to update

store code inside the account leaf

fix build

save current state for Sina

Update go-verkle to latest

Charge WITNESS_*_COST gas on storage loads

Add witness costs for SSTORE as well

Charge witness gas in the case of code execution

corresponding code deletion

add a --verkle flag to separate verkle experiments from regular geth operations

use the snapshot to get data

stateless execution from block witness

AccessWitness functions

Add block generation test + genesis snapshot generation

test stateless block execution (#18)

* test stateless block execution

* Force tree resolution before generating the proof

increased coverage in stateless test execution (#19)

* test stateless block execution

* Force tree resolution before generating the proof

* increase coverage in stateless test execution

ensure geth compiles

fix issues in tests with verkle trees deactivated

Ensure stateless data is available when executing statelessly (#20)

* Ensure stateless data is available when executing statelessly

* Actual execution of a statless block

* bugfixes in stateless block execution

* code cleanup

 - Reduce PR footprint by reverting NewEVM to its original signature
 - Move the access witness to the block context
 - prepare for a change in AW semantics
   Need to store the initial values.
 - Use the touch helper function, DRY

* revert the signature of MustCommit to its original form (#21)

fix leaf proofs in stateless execution (#22)

* Fixes in witness pre-state

* Add the recipient's nonce to the witness

* reduce PR footprint and investigate issue in root state calculation

* quick build fix

cleanup: Remove extra parameter in ToBlock

revert ToBlock to its older signature

Author: gballet

cmd/geth/main.go

@@ -150,6 +150,7 @@ var (
 		utils.MinerNotifyFullFlag,
 		configFileFlag,
 		utils.CatalystFlag,
+		utils.VerkleFlag,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/geth/snapshot.go

@@ -215,7 +215,7 @@ func verifyState(ctx *cli.Context) error {
 		log.Error("Failed to load head block")
 		return errors.New("no head block")
 	}
-	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false)
+	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false, false)
 	if err != nil {
 		log.Error("Failed to open snapshot tree", "err", err)
 		return err
@@ -467,7 +467,7 @@ func dumpState(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false)
+	snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false, false)
 	if err != nil {
 		return err
 	}

cmd/geth/usage.go

@@ -227,6 +227,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.BloomFilterSizeFlag,
 			cli.HelpFlag,
 			utils.CatalystFlag,
+			utils.VerkleFlag,
 		},
 	},
 }

cmd/utils/flags.go

@@ -775,6 +775,11 @@ var (
 		Name:  "catalyst",
 		Usage: "Catalyst mode (eth2 integration testing)",
 	}
+
+	VerkleFlag = cli.BoolFlag{
+		Name:  "verkle",
+		Usage: "Enable geth with verkle trees (EXPERIMENTAL)",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1449,7 +1454,7 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) {
 // SetEthConfig applies eth-related command line flags to the config.
 func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	// Avoid conflicting network flags
-	CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag)
+	CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, VerkleFlag)
 	CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light")
 	CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
 	if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 {
@@ -1584,6 +1589,13 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		}
 		cfg.Genesis = core.DefaultGenesisBlock()
 		SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash)
+	case ctx.GlobalBool(VerkleFlag.Name):
+		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
+			cfg.NetworkId = 86 // 'V'
+		}
+		cfg.Genesis = core.DefaultGenesisBlock()
+		cfg.Genesis.Config.UseVerkle = true
+		SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash)
 	case ctx.GlobalBool(RopstenFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 3

core/blockchain.go

@@ -233,6 +233,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 			Cache:     cacheConfig.TrieCleanLimit,
 			Journal:   cacheConfig.TrieCleanJournal,
 			Preimages: cacheConfig.Preimages,
+			UseVerkle: chainConfig.UseVerkle,
 		}),
 		quit:           make(chan struct{}),
 		shouldPreserve: shouldPreserve,
@@ -370,7 +371,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 			log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer)
 			recover = true
 		}
-		bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, head.Root(), !bc.cacheConfig.SnapshotWait, true, recover)
+		bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, head.Root(), !bc.cacheConfig.SnapshotWait, true, recover, bc.Config().UseVerkle)
 	}
 	// Take ownership of this particular state
 	go bc.update()
@@ -1825,7 +1826,22 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
 		}
 		// Process block using the parent state as reference point
 		substart := time.Now()
-		receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig)
+		var (
+			usedGas  uint64
+			receipts types.Receipts
+			logs     []*types.Log
+		)
+		if len(block.Header().VerkleProof) == 0 {
+			receipts, logs, _, usedGas, err = bc.processor.Process(block, statedb, bc.vmConfig)
+		} else {
+			var leaves map[common.Hash]common.Hash
+			_, _, _, leaves, err = trie.DeserializeVerkleProof(block.Header().VerkleProof)
+			if err != nil {
+				return it.index, err
+			}
+			statedb.SetStateless(leaves)
+			receipts, logs, usedGas, err = bc.processor.ProcessStateless(block, statedb, bc.vmConfig, leaves)
+		}
 		if err != nil {
 			bc.reportBlock(block, receipts, err)
 			atomic.StoreUint32(&followupInterrupt, 1)

core/blockchain_test.go

@@ -153,7 +153,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
 		if err != nil {
 			return err
 		}
-		receipts, _, usedGas, err := blockchain.processor.Process(block, statedb, vm.Config{})
+		receipts, _, _, usedGas, err := blockchain.processor.Process(block, statedb, vm.Config{})
 		if err != nil {
 			blockchain.reportBlock(block, receipts, err)
 			return err

core/chain_makers.go

@@ -28,6 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // BlockGen creates blocks for testing.
@@ -43,6 +44,7 @@ type BlockGen struct {
 	txs      []*types.Transaction
 	receipts []*types.Receipt
 	uncles   []*types.Header
+	witness  *types.AccessWitness
 
 	config *params.ChainConfig
 	engine consensus.Engine
@@ -103,10 +105,17 @@ func (b *BlockGen) AddTxWithChain(bc *BlockChain, tx *types.Transaction) {
 		b.SetCoinbase(common.Address{})
 	}
 	b.statedb.Prepare(tx.Hash(), len(b.txs))
-	receipt, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{})
+	receipt, accesses, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{})
 	if err != nil {
 		panic(err)
 	}
+	if accesses != nil {
+		if b.witness != nil {
+			b.witness.Merge(accesses)
+		} else {
+			b.witness = accesses
+		}
+	}
 	b.txs = append(b.txs, tx)
 	b.receipts = append(b.receipts, receipt)
 }
@@ -250,6 +259,90 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 	return blocks, receipts
 }
 
+func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) {
+	if config == nil {
+		config = params.TestChainConfig
+	}
+	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
+	chainreader := &fakeChainReader{config: config}
+	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
+		b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine, witness: types.NewAccessWitness()}
+		b.header = makeHeader(chainreader, parent, statedb, b.engine)
+
+		// Mutate the state and block according to any hard-fork specs
+		if daoBlock := config.DAOForkBlock; daoBlock != nil {
+			limit := new(big.Int).Add(daoBlock, params.DAOForkExtraRange)
+			if b.header.Number.Cmp(daoBlock) >= 0 && b.header.Number.Cmp(limit) < 0 {
+				if config.DAOForkSupport {
+					b.header.Extra = common.CopyBytes(params.DAOForkBlockExtra)
+				}
+			}
+		}
+		if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(b.header.Number) == 0 {
+			misc.ApplyDAOHardFork(statedb)
+		}
+		// Execute any user modifications to the block
+		if gen != nil {
+			gen(i, b)
+		}
+		if b.engine != nil {
+			// Finalize and seal the block
+			block, err := b.engine.FinalizeAndAssemble(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts)
+			if err != nil {
+				panic(err)
+			}
+
+			// Write state changes to db
+			root, err := statedb.Commit(config.IsEIP158(b.header.Number))
+			if err != nil {
+				panic(fmt.Sprintf("state write error: %v", err))
+			}
+			if err := statedb.Database().TrieDB().Commit(root, false, nil); err != nil {
+				panic(fmt.Sprintf("trie write error: %v", err))
+			}
+
+			// Generate an associated verkle proof
+			if tr := statedb.GetTrie(); tr.IsVerkle() {
+				vtr := tr.(*trie.VerkleTrie)
+				// Generate the proof if we are using a verkle tree
+				// WORKAROUND: make sure all keys are resolved
+				// before building the proof. Ultimately, node
+				// resolution can be done with a prefetcher or
+				// from GetCommitmentsAlongPath.
+				keys := b.witness.Keys()
+				for _, key := range keys {
+					out, err := vtr.TryGet(key)
+					if err != nil {
+						panic(err)
+					}
+					if len(out) == 0 {
+						panic(fmt.Sprintf("%x should be present in the tree", key))
+					}
+				}
+				vtr.Hash()
+				_, err := vtr.ProveAndSerialize(keys, b.witness.KeyVals())
+				//block.SetVerkleProof(p)
+				if err != nil {
+					panic(err)
+				}
+			}
+			return block, b.receipts
+		}
+		return nil, nil
+	}
+	for i := 0; i < n; i++ {
+		statedb, err := state.New(parent.Root(), state.NewDatabaseWithConfig(db, &trie.Config{UseVerkle: true}), nil)
+		if err != nil {
+			panic(err)
+		}
+		block, receipt := genblock(i, parent, statedb)
+		blocks[i] = block
+		receipts[i] = receipt
+		parent = block
+	}
+	return blocks, receipts
+}
+
 func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header {
 	var time uint64
 	if parent.Time() == 0 {

core/evm.go

@@ -69,6 +69,7 @@ func NewEVMTxContext(msg Message) vm.TxContext {
 	return vm.TxContext{
 		Origin:   msg.From(),
 		GasPrice: new(big.Int).Set(msg.GasPrice()),
+		Accesses: types.NewAccessWitness(),
 	}
 }
 

core/genesis.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -256,10 +257,18 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
 // ToBlock creates the genesis block and writes state of a genesis specification
 // to the given database (or discards it if nil).
 func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
+	return g.ToBlockWithSnaps(db, nil)
+}
+
+func (g *Genesis) ToBlockWithSnaps(db ethdb.Database, snaps *snapshot.Tree) *types.Block {
 	if db == nil {
 		db = rawdb.NewMemoryDatabase()
 	}
-	statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil)
+	var trieCfg *trie.Config
+	if g.Config != nil {
+		trieCfg = &trie.Config{UseVerkle: g.Config.UseVerkle}
+	}
+	statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, trieCfg), snaps)
 	if err != nil {
 		panic(err)
 	}
@@ -301,14 +310,21 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
 	}
 	statedb.Commit(false)
 	statedb.Database().TrieDB().Commit(root, true, nil)
+	if err := statedb.Cap(root); err != nil {
+		panic(err)
+	}
 
 	return types.NewBlock(head, nil, nil, nil, trie.NewStackTrie(nil))
 }
 
 // Commit writes the block and state of a genesis specification to the database.
 // The block is committed as the canonical head block.
 func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) {
-	block := g.ToBlock(db)
+	return g.CommitWithSnaps(db, nil)
+}
+
+func (g *Genesis) CommitWithSnaps(db ethdb.Database, snaps *snapshot.Tree) (*types.Block, error) {
+	block := g.ToBlockWithSnaps(db, snaps)
 	if block.Number().Sign() != 0 {
 		return nil, errors.New("can't commit genesis block with number > 0")
 	}