Skip to content

Commit 23dd9eb

Browse files
rjl493456442holimankaralabe
authored and
atif-konasl
committed
core, eth: faster snapshot generation (ethereum#22504)
* eth/protocols: persist received state segments * core: initial implementation * core/state/snapshot: add tests * core, eth: updates * eth/protocols/snapshot: count flat state size * core/state: add metrics * core/state/snapshot: skip unnecessary deletion * core/state/snapshot: rename * core/state/snapshot: use the global batch * core/state/snapshot: add logs and fix wiping * core/state/snapshot: fix * core/state/snapshot: save generation progress even if the batch is empty * core/state/snapshot: fixes * core/state/snapshot: fix initial account range length * core/state/snapshot: fix initial account range * eth/protocols/snap: store flat states during the healing * eth/protocols/snap: print logs * core/state/snapshot: refactor (#4) * core/state/snapshot: refactor * core/state/snapshot: tiny fix and polish Co-authored-by: rjl493456442 <[email protected]> * core, eth: fixes * core, eth: fix healing writer * core, trie, eth: fix paths * eth/protocols/snap: fix encoding * eth, core: add debug log * core/state/generate: release iterator asap (#5) core/state/snapshot: less copy core/state/snapshot: revert split loop core/state/snapshot: handle storage becoming empty, improve test robustness core/state: test modified codehash core/state/snapshot: polish * core/state/snapshot: optimize stats counter * core, eth: add metric * core/state/snapshot: update comments * core/state/snapshot: improve tests * core/state/snapshot: replace secure trie with standard trie * core/state/snapshot: wrap return as the struct * core/state/snapshot: skip wiping correct states * core/state/snapshot: updates * core/state/snapshot: fixes * core/state/snapshot: fix panic due to reference flaw in closure * core/state/snapshot: fix errors in state generation logic + fix log output * core/state/snapshot: remove an error case * core/state/snapshot: fix condition-check for exhausted snap state * core/state/snapshot: use stackTrie for small tries * core/state/snapshot: don't resolve small storage tries in vain * core/state/snapshot: properly clean up storage of deleted accounts * core/state/snapshot: avoid RLP-encoding in some cases + minor nitpicks * core/state/snapshot: fix error (+testcase) * core/state/snapshot: clean up tests a bit * core/state/snapshot: work in progress on better tests * core/state/snapshot: polish code * core/state/snapshot: fix trie iteration abortion trigger * core/state/snapshot: fixes flaws * core/state/snapshot: remove panic * core/state/snapshot: fix abort * core/state/snapshot: more tests (plus failing testcase) * core/state/snapshot: more testcases + fix for failing test * core/state/snapshot: testcase for malformed data * core/state/snapshot: some test nitpicks * core/state/snapshot: improvements to logging * core/state/snapshot: testcase to demo error in abortion * core/state/snapshot: fix abortion * cmd/geth: make verify-state report the root * trie: fix failing test * core/state/snapshot: add timer metrics * core/state/snapshot: fix metrics * core/state/snapshot: udpate tests * eth/protocols/snap: write snapshot account even if code or state is needed * core/state/snapshot: fix diskmore check * core/state/snapshot: review fixes * core/state/snapshot: improve error message * cmd/geth: rename 'error' to 'err' in logs * core/state/snapshot: fix some review concerns * core/state/snapshot, eth/protocols/snap: clear snapshot marker when starting/resuming snap sync * core: add error log * core/state/snapshot: use proper timers for metrics collection * core/state/snapshot: address some review concerns * eth/protocols/snap: improved log message * eth/protocols/snap: fix heal logs to condense infos * core/state/snapshot: wait for generator termination before restarting * core/state/snapshot: revert timers to counters to track total time Co-authored-by: Martin Holst Swende <[email protected]> Co-authored-by: Péter Szilágyi <[email protected]>
1 parent 0d42d32 commit 23dd9eb

File tree

16 files changed

+1369
-209
lines changed

16 files changed

+1369
-209
lines changed

‎cmd/geth/snapshot.go

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ func pruneState(ctx *cli.Context) error {
155155
chaindb := utils.MakeChainDatabase(ctx, stack, false)
156156
pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name))
157157
if err != nil {
158-
log.Error("Failed to open snapshot tree", "error", err)
158+
log.Error("Failed to open snapshot tree", "err", err)
159159
return err
160160
}
161161
if ctx.NArg() > 1 {
@@ -166,12 +166,12 @@ func pruneState(ctx *cli.Context) error {
166166
if ctx.NArg() == 1 {
167167
targetRoot, err = parseRoot(ctx.Args()[0])
168168
if err != nil {
169-
log.Error("Failed to resolve state root", "error", err)
169+
log.Error("Failed to resolve state root", "err", err)
170170
return err
171171
}
172172
}
173173
if err = pruner.Prune(targetRoot); err != nil {
174-
log.Error("Failed to prune state", "error", err)
174+
log.Error("Failed to prune state", "err", err)
175175
return err
176176
}
177177
return nil
@@ -189,7 +189,7 @@ func verifyState(ctx *cli.Context) error {
189189
}
190190
snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false)
191191
if err != nil {
192-
log.Error("Failed to open snapshot tree", "error", err)
192+
log.Error("Failed to open snapshot tree", "err", err)
193193
return err
194194
}
195195
if ctx.NArg() > 1 {
@@ -200,15 +200,15 @@ func verifyState(ctx *cli.Context) error {
200200
if ctx.NArg() == 1 {
201201
root, err = parseRoot(ctx.Args()[0])
202202
if err != nil {
203-
log.Error("Failed to resolve state root", "error", err)
203+
log.Error("Failed to resolve state root", "err", err)
204204
return err
205205
}
206206
}
207207
if err := snaptree.Verify(root); err != nil {
208-
log.Error("Failed to verfiy state", "error", err)
208+
log.Error("Failed to verfiy state", "root", root, "err", err)
209209
return err
210210
}
211-
log.Info("Verified the state")
211+
log.Info("Verified the state", "root", root)
212212
return nil
213213
}
214214

@@ -236,7 +236,7 @@ func traverseState(ctx *cli.Context) error {
236236
if ctx.NArg() == 1 {
237237
root, err = parseRoot(ctx.Args()[0])
238238
if err != nil {
239-
log.Error("Failed to resolve state root", "error", err)
239+
log.Error("Failed to resolve state root", "err", err)
240240
return err
241241
}
242242
log.Info("Start traversing the state", "root", root)
@@ -247,7 +247,7 @@ func traverseState(ctx *cli.Context) error {
247247
triedb := trie.NewDatabase(chaindb)
248248
t, err := trie.NewSecure(root, triedb)
249249
if err != nil {
250-
log.Error("Failed to open trie", "root", root, "error", err)
250+
log.Error("Failed to open trie", "root", root, "err", err)
251251
return err
252252
}
253253
var (
@@ -262,21 +262,21 @@ func traverseState(ctx *cli.Context) error {
262262
accounts += 1
263263
var acc state.Account
264264
if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil {
265-
log.Error("Invalid account encountered during traversal", "error", err)
265+
log.Error("Invalid account encountered during traversal", "err", err)
266266
return err
267267
}
268268
if acc.Root != emptyRoot {
269269
storageTrie, err := trie.NewSecure(acc.Root, triedb)
270270
if err != nil {
271-
log.Error("Failed to open storage trie", "root", acc.Root, "error", err)
271+
log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
272272
return err
273273
}
274274
storageIter := trie.NewIterator(storageTrie.NodeIterator(nil))
275275
for storageIter.Next() {
276276
slots += 1
277277
}
278278
if storageIter.Err != nil {
279-
log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Err)
279+
log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err)
280280
return storageIter.Err
281281
}
282282
}
@@ -294,7 +294,7 @@ func traverseState(ctx *cli.Context) error {
294294
}
295295
}
296296
if accIter.Err != nil {
297-
log.Error("Failed to traverse state trie", "root", root, "error", accIter.Err)
297+
log.Error("Failed to traverse state trie", "root", root, "err", accIter.Err)
298298
return accIter.Err
299299
}
300300
log.Info("State is complete", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
@@ -326,7 +326,7 @@ func traverseRawState(ctx *cli.Context) error {
326326
if ctx.NArg() == 1 {
327327
root, err = parseRoot(ctx.Args()[0])
328328
if err != nil {
329-
log.Error("Failed to resolve state root", "error", err)
329+
log.Error("Failed to resolve state root", "err", err)
330330
return err
331331
}
332332
log.Info("Start traversing the state", "root", root)
@@ -337,7 +337,7 @@ func traverseRawState(ctx *cli.Context) error {
337337
triedb := trie.NewDatabase(chaindb)
338338
t, err := trie.NewSecure(root, triedb)
339339
if err != nil {
340-
log.Error("Failed to open trie", "root", root, "error", err)
340+
log.Error("Failed to open trie", "root", root, "err", err)
341341
return err
342342
}
343343
var (
@@ -368,13 +368,13 @@ func traverseRawState(ctx *cli.Context) error {
368368
accounts += 1
369369
var acc state.Account
370370
if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil {
371-
log.Error("Invalid account encountered during traversal", "error", err)
371+
log.Error("Invalid account encountered during traversal", "err", err)
372372
return errors.New("invalid account")
373373
}
374374
if acc.Root != emptyRoot {
375375
storageTrie, err := trie.NewSecure(acc.Root, triedb)
376376
if err != nil {
377-
log.Error("Failed to open storage trie", "root", acc.Root, "error", err)
377+
log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
378378
return errors.New("missing storage trie")
379379
}
380380
storageIter := storageTrie.NodeIterator(nil)
@@ -397,7 +397,7 @@ func traverseRawState(ctx *cli.Context) error {
397397
}
398398
}
399399
if storageIter.Error() != nil {
400-
log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Error())
400+
log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error())
401401
return storageIter.Error()
402402
}
403403
}
@@ -416,7 +416,7 @@ func traverseRawState(ctx *cli.Context) error {
416416
}
417417
}
418418
if accIter.Error() != nil {
419-
log.Error("Failed to traverse state trie", "root", root, "error", accIter.Error())
419+
log.Error("Failed to traverse state trie", "root", root, "err", accIter.Error())
420420
return accIter.Error()
421421
}
422422
log.Info("State is complete", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))

‎core/state/snapshot/conversion.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash,
322322
return
323323
}
324324
if !bytes.Equal(account.Root, subroot.Bytes()) {
325-
results <- fmt.Errorf("invalid subroot(%x), want %x, got %x", it.Hash(), account.Root, subroot)
325+
results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, account.Root, subroot)
326326
return
327327
}
328328
results <- nil

0 commit comments

Comments
 (0)