Skip to content

Commit 9e4f4f3

Browse files
tsaheetsahee
committed
Merge upstream PR ethereum#27971
Conflicts: core/blockchain.go We changed writeBlockWithState, and upstream added a return value to TrieDB().Size() - that extra value is ignored both upstream and after merge. Additions: arbitrum/recordingdb.go The same change for TrieDB().Size()
2 parents 12679bb + 0c6bbeb commit 9e4f4f3

28 files changed

+107
-69
lines changed

‎arbitrum/recordingdb.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -248,13 +248,13 @@ func (r *RecordingDatabase) addStateVerify(statedb *state.StateDB, expected comm
248248
}
249249
r.referenceRootLockHeld(result)
250250

251-
size, _ := r.db.TrieDB().Size()
251+
_, size, _ := r.db.TrieDB().Size()
252252
limit := common.StorageSize(r.config.TrieDirtyCache) * 1024 * 1024
253253
recordingDbSize.Update(int64(size))
254254
if size > limit {
255255
log.Info("Recording DB: flushing to disk", "size", size, "limit", limit)
256256
r.db.TrieDB().Cap(limit - ethdb.IdealBatchSize)
257-
size, _ = r.db.TrieDB().Size()
257+
_, size, _ = r.db.TrieDB().Size()
258258
recordingDbSize.Update(int64(size))
259259
}
260260
return state.New(result, statedb.Database(), nil)

‎cmd/evm/staterunner.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -107,19 +107,20 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error {
107107
// Run the test and aggregate the result
108108
result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
109109
test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
110+
if state != nil {
111+
root := state.IntermediateRoot(false)
112+
result.Root = &root
113+
if jsonOut {
114+
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
115+
}
116+
}
110117
if err != nil {
111118
// Test failed, mark as so and dump any state to aid debugging
112119
result.Pass, result.Error = false, err.Error()
113120
if dump {
114121
dump := state.RawDump(nil)
115122
result.State = &dump
116123
}
117-
} else {
118-
root := state.IntermediateRoot(false)
119-
result.Root = &root
120-
if jsonOut {
121-
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
122-
}
123124
}
124125
})
125126
results = append(results, *result)

‎core/blockchain.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1100,7 +1100,7 @@ func (bc *BlockChain) Stop() {
11001100
for !bc.triegc.Empty() {
11011101
triedb.Dereference(bc.triegc.PopItem().Root)
11021102
}
1103-
if size, _ := triedb.Size(); size != 0 {
1103+
if _, nodes, _ := triedb.Size(); nodes != 0 { // all memory is contained within the nodes return for hashdb
11041104
log.Error("Dangling trie nodes after full cleanup")
11051105
}
11061106
}
@@ -1534,8 +1534,8 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
15341534
if blockLimit > 0 && timeLimit > 0 {
15351535
// If we exceeded our memory allowance, flush matured singleton nodes to disk
15361536
var (
1537-
nodes, imgs = bc.triedb.Size()
1538-
limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
1537+
_, nodes, imgs = bc.triedb.Size() // all memory is contained within the nodes return for hashdb
1538+
limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
15391539
)
15401540
if nodes > limit || imgs > 4*1024*1024 {
15411541
bc.triedb.Cap(limit - ethdb.IdealBatchSize)
@@ -1980,8 +1980,12 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
19801980
stats.processed++
19811981
stats.usedGas += usedGas
19821982

1983-
dirty, _ := bc.triedb.Size()
1984-
stats.report(chain, it.index, dirty, setHead)
1983+
var snapDiffItems, snapBufItems common.StorageSize
1984+
if bc.snaps != nil {
1985+
snapDiffItems, snapBufItems = bc.snaps.Size()
1986+
}
1987+
trieDiffNodes, trieBufNodes, _ := bc.triedb.Size()
1988+
stats.report(chain, it.index, snapDiffItems, snapBufItems, trieDiffNodes, trieBufNodes, setHead)
19851989

19861990
if !setHead {
19871991
// After merge we expect few side chains. Simply count

‎core/blockchain_insert.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ const statsReportLimit = 8 * time.Second
3939

4040
// report prints statistics if some number of blocks have been processed
4141
// or more than a few seconds have passed since the last message.
42-
func (st *insertStats) report(chain []*types.Block, index int, dirty common.StorageSize, setHead bool) {
42+
func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, snapBufItems, trieDiffNodes, triebufNodes common.StorageSize, setHead bool) {
4343
// Fetch the timings for the batch
4444
var (
4545
now = mclock.Now()
@@ -63,7 +63,16 @@ func (st *insertStats) report(chain []*types.Block, index int, dirty common.Stor
6363
if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute {
6464
context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
6565
}
66-
context = append(context, []interface{}{"dirty", dirty}...)
66+
if snapDiffItems != 0 || snapBufItems != 0 { // snapshots enabled
67+
context = append(context, []interface{}{"snapdiffs", snapDiffItems}...)
68+
if snapBufItems != 0 { // future snapshot refactor
69+
context = append(context, []interface{}{"snapdirty", snapBufItems}...)
70+
}
71+
}
72+
if trieDiffNodes != 0 { // pathdb
73+
context = append(context, []interface{}{"triediffs", trieDiffNodes}...)
74+
}
75+
context = append(context, []interface{}{"triedirty", triebufNodes}...)
6776

6877
if st.queued > 0 {
6978
context = append(context, []interface{}{"queued", st.queued}...)

‎core/blockchain_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1844,7 +1844,7 @@ func TestTrieForkGC(t *testing.T) {
18441844
chain.TrieDB().Dereference(blocks[len(blocks)-1-i].Root())
18451845
chain.TrieDB().Dereference(forks[len(blocks)-1-i].Root())
18461846
}
1847-
if nodes, _ := chain.TrieDB().Size(); nodes > 0 {
1847+
if _, nodes, _ := chain.TrieDB().Size(); nodes > 0 { // all memory is returned in the nodes return for hashdb
18481848
t.Fatalf("stale tries still alive after garbase collection")
18491849
}
18501850
}

‎core/chain_makers.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,7 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S
411411
excessBlobGas := eip4844.CalcExcessBlobGas(parentExcessBlobGas, parentBlobGasUsed)
412412
header.ExcessBlobGas = &excessBlobGas
413413
header.BlobGasUsed = new(uint64)
414+
header.BeaconRoot = new(common.Hash)
414415
}
415416
return header
416417
}

‎core/state/snapshot/snapshot.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -858,3 +858,21 @@ func (t *Tree) DiskRoot() common.Hash {
858858

859859
return t.diskRoot()
860860
}
861+
862+
// Size returns the memory usage of the diff layers above the disk layer and the
863+
// dirty nodes buffered in the disk layer. Currently, the implementation uses a
864+
// special diff layer (the first) as an aggregator simulating a dirty buffer, so
865+
// the second return will always be 0. However, this will be made consistent with
866+
// the pathdb, which will require a second return.
867+
func (t *Tree) Size() (diffs common.StorageSize, buf common.StorageSize) {
868+
t.lock.RLock()
869+
defer t.lock.RUnlock()
870+
871+
var size common.StorageSize
872+
for _, layer := range t.layers {
873+
if layer, ok := layer.(*diffLayer); ok {
874+
size += common.StorageSize(layer.memory)
875+
}
876+
}
877+
return size, 0
878+
}

‎eth/state_accessor.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u
168168
parent = root
169169
}
170170
if report {
171-
nodes, imgs := triedb.Size()
171+
_, nodes, imgs := triedb.Size() // all memory is contained within the nodes return in hashdb
172172
log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs)
173173
}
174174
return statedb, func() { triedb.Dereference(block.Root()) }, nil

‎eth/tracers/api.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -369,8 +369,8 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed
369369
// if the relevant state is available in disk.
370370
var preferDisk bool
371371
if statedb != nil {
372-
s1, s2 := statedb.Database().TrieDB().Size()
373-
preferDisk = s1+s2 > defaultTracechainMemLimit
372+
s1, s2, s3 := statedb.Database().TrieDB().Size()
373+
preferDisk = s1+s2+s3 > defaultTracechainMemLimit
374374
}
375375
statedb, release, err = api.backend.StateAtBlock(ctx, block, reexec, statedb, false, preferDisk)
376376
if err != nil {

‎go.mod

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.20
44

55
require (
66
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0
7+
github.com/Microsoft/go-winio v0.6.1
78
github.com/VictoriaMetrics/fastcache v1.6.0
89
github.com/aws/aws-sdk-go-v2 v1.2.0
910
github.com/aws/aws-sdk-go-v2/config v1.1.1
@@ -12,7 +13,7 @@ require (
1213
github.com/btcsuite/btcd/btcec/v2 v2.2.0
1314
github.com/cespare/cp v0.1.0
1415
github.com/cloudflare/cloudflare-go v0.14.0
15-
github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811
16+
github.com/cockroachdb/pebble v0.0.0-20230821143352-55b44ac08de8
1617
github.com/consensys/gnark-crypto v0.10.0
1718
github.com/crate-crypto/go-kzg-4844 v0.3.0
1819
github.com/davecgh/go-spew v1.1.1
@@ -73,7 +74,6 @@ require (
7374
golang.org/x/time v0.3.0
7475
golang.org/x/tools v0.9.1
7576
gopkg.in/natefinch/lumberjack.v2 v2.0.0
76-
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
7777
gopkg.in/yaml.v3 v3.0.1
7878
)
7979

@@ -132,5 +132,6 @@ require (
132132
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
133133
google.golang.org/protobuf v1.28.1 // indirect
134134
gopkg.in/yaml.v2 v2.4.0 // indirect
135+
gotest.tools/v3 v3.5.0 // indirect
135136
rsc.io/tmplfunc v0.0.3 // indirect
136137
)

0 commit comments

Comments
 (0)