Skip to content

Commit caea6c4

Browse files
karalabeholiman
karalabe
and
holiman
authored
eth/protocols/snap: generate storage trie from full dirty snap data (#22668)
* eth/protocols/snap: generate storage trie from full dirty snap data * eth/protocols/snap: get rid of some more dead code * eth/protocols/snap: less frequent logs, also log during trie generation * eth/protocols/snap: implement dirty account range stack-hashing * eth/protocols/snap: don't loop on account trie generation * eth/protocols/snap: fix account format in trie * core, eth, ethdb: glue snap packets together, but not chunks * eth/protocols/snap: print completion log for snap phase * eth/protocols/snap: extended tests * eth/protocols/snap: make testcase pass * eth/protocols/snap: fix account stacktrie commit without defer * ethdb: fix key counts on reset * eth/protocols: fix typos * eth/protocols/snap: make better use of delivered data (#44) * eth/protocols/snap: make better use of delivered data * squashme * eth/protocols/snap: reduce chunking * squashme * eth/protocols/snap: reduce chunking further * eth/protocols/snap: break out hash range calculations * eth/protocols/snap: use sort.Search instead of looping * eth/protocols/snap: prevent crash on storage response with no keys * eth/protocols/snap: nitpicks all around * eth/protocols/snap: clear heal need on 1-chunk storage completion * eth/protocols/snap: fix range chunker, add tests Co-authored-by: Péter Szilágyi <[email protected]> * trie: fix test API error * eth/protocols/snap: fix some further liter issues * eth/protocols/snap: fix accidental batch reuse Co-authored-by: Martin Holst Swende <[email protected]>
1 parent 65a1c2d commit caea6c4

File tree

12 files changed

+603
-188
lines changed

12 files changed

+603
-188
lines changed

‎core/rawdb/database_test.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright 2019 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package rawdb

‎core/rawdb/table.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,11 @@ func (b *tableBatch) Delete(key []byte) error {
176176
return b.batch.Delete(append([]byte(b.prefix), key...))
177177
}
178178

179+
// KeyCount retrieves the number of keys queued up for writing.
180+
func (b *tableBatch) KeyCount() int {
181+
return b.batch.KeyCount()
182+
}
183+
179184
// ValueSize retrieves the amount of data queued up for writing.
180185
func (b *tableBatch) ValueSize() int {
181186
return b.batch.ValueSize()

‎eth/protocols/snap/handler.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ func handleMessage(backend Backend, peer *Peer) error {
354354
if err := msg.Decode(res); err != nil {
355355
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
356356
}
357-
// Ensure the ranges ae monotonically increasing
357+
// Ensure the ranges are monotonically increasing
358358
for i, slots := range res.Slots {
359359
for j := 1; j < len(slots); j++ {
360360
if bytes.Compare(slots[j-1].Hash[:], slots[j].Hash[:]) >= 0 {

‎eth/protocols/snap/range.go

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright 2021 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package snap
18+
19+
import (
20+
"math/big"
21+
22+
"github.com/ethereum/go-ethereum/common"
23+
"github.com/holiman/uint256"
24+
)
25+
26+
// hashRange is a utility to handle ranges of hashes, Split up the
27+
// hash-space into sections, and 'walk' over the sections
28+
type hashRange struct {
29+
current *uint256.Int
30+
step *uint256.Int
31+
}
32+
33+
// newHashRange creates a new hashRange, initiated at the start position,
34+
// and with the step set to fill the desired 'num' chunks
35+
func newHashRange(start common.Hash, num uint64) *hashRange {
36+
left := new(big.Int).Sub(hashSpace, start.Big())
37+
step := new(big.Int).Div(
38+
new(big.Int).Add(left, new(big.Int).SetUint64(num-1)),
39+
new(big.Int).SetUint64(num),
40+
)
41+
step256 := new(uint256.Int)
42+
step256.SetFromBig(step)
43+
44+
return &hashRange{
45+
current: uint256.NewInt().SetBytes32(start[:]),
46+
step: step256,
47+
}
48+
}
49+
50+
// Next pushes the hash range to the next interval.
51+
func (r *hashRange) Next() bool {
52+
next := new(uint256.Int)
53+
if overflow := next.AddOverflow(r.current, r.step); overflow {
54+
return false
55+
}
56+
r.current = next
57+
return true
58+
}
59+
60+
// Start returns the first hash in the current interval.
61+
func (r *hashRange) Start() common.Hash {
62+
return r.current.Bytes32()
63+
}
64+
65+
// End returns the last hash in the current interval.
66+
func (r *hashRange) End() common.Hash {
67+
// If the end overflows (non divisible range), return a shorter interval
68+
next := new(uint256.Int)
69+
if overflow := next.AddOverflow(r.current, r.step); overflow {
70+
return common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
71+
}
72+
return new(uint256.Int).Sub(next, uint256.NewInt().SetOne()).Bytes32()
73+
}
74+
75+
// incHash returns the next hash, in lexicographical order (a.k.a plus one)
76+
func incHash(h common.Hash) common.Hash {
77+
a := uint256.NewInt().SetBytes32(h[:])
78+
a.Add(a, uint256.NewInt().SetOne())
79+
return common.Hash(a.Bytes32())
80+
}

‎eth/protocols/snap/range_test.go

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
// Copyright 2021 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package snap
18+
19+
import (
20+
"testing"
21+
22+
"github.com/ethereum/go-ethereum/common"
23+
)
24+
25+
// Tests that given a starting hash and a density, the hash ranger can correctly
26+
// split up the remaining hash space into a fixed number of chunks.
27+
func TestHashRanges(t *testing.T) {
28+
tests := []struct {
29+
head common.Hash
30+
chunks uint64
31+
starts []common.Hash
32+
ends []common.Hash
33+
}{
34+
// Simple test case to split the entire hash range into 4 chunks
35+
{
36+
head: common.Hash{},
37+
chunks: 4,
38+
starts: []common.Hash{
39+
{},
40+
common.HexToHash("0x4000000000000000000000000000000000000000000000000000000000000000"),
41+
common.HexToHash("0x8000000000000000000000000000000000000000000000000000000000000000"),
42+
common.HexToHash("0xc000000000000000000000000000000000000000000000000000000000000000"),
43+
},
44+
ends: []common.Hash{
45+
common.HexToHash("0x3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
46+
common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
47+
common.HexToHash("0xbfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
48+
common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
49+
},
50+
},
51+
// Split a divisible part of the hash range up into 2 chunks
52+
{
53+
head: common.HexToHash("0x2000000000000000000000000000000000000000000000000000000000000000"),
54+
chunks: 2,
55+
starts: []common.Hash{
56+
common.Hash{},
57+
common.HexToHash("0x9000000000000000000000000000000000000000000000000000000000000000"),
58+
},
59+
ends: []common.Hash{
60+
common.HexToHash("0x8fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
61+
common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
62+
},
63+
},
64+
// Split the entire hash range into a non divisible 3 chunks
65+
{
66+
head: common.Hash{},
67+
chunks: 3,
68+
starts: []common.Hash{
69+
{},
70+
common.HexToHash("0x5555555555555555555555555555555555555555555555555555555555555556"),
71+
common.HexToHash("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"),
72+
},
73+
ends: []common.Hash{
74+
common.HexToHash("0x5555555555555555555555555555555555555555555555555555555555555555"),
75+
common.HexToHash("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"),
76+
common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
77+
},
78+
},
79+
// Split a part of hash range into a non divisible 3 chunks
80+
{
81+
head: common.HexToHash("0x2000000000000000000000000000000000000000000000000000000000000000"),
82+
chunks: 3,
83+
starts: []common.Hash{
84+
{},
85+
common.HexToHash("0x6aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"),
86+
common.HexToHash("0xb555555555555555555555555555555555555555555555555555555555555556"),
87+
},
88+
ends: []common.Hash{
89+
common.HexToHash("0x6aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
90+
common.HexToHash("0xb555555555555555555555555555555555555555555555555555555555555555"),
91+
common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
92+
},
93+
},
94+
// Split a part of hash range into a non divisible 3 chunks, but with a
95+
// meaningful space size for manual verification.
96+
// - The head being 0xff...f0, we have 14 hashes left in the space
97+
// - Chunking up 14 into 3 pieces is 4.(6), but we need the ceil of 5 to avoid a micro-last-chunk
98+
// - Since the range is not divisible, the last interval will be shrter, capped at 0xff...f
99+
// - The chunk ranges thus needs to be [..0, ..5], [..6, ..b], [..c, ..f]
100+
{
101+
head: common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0"),
102+
chunks: 3,
103+
starts: []common.Hash{
104+
{},
105+
common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6"),
106+
common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc"),
107+
},
108+
ends: []common.Hash{
109+
common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5"),
110+
common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb"),
111+
common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
112+
},
113+
},
114+
}
115+
for i, tt := range tests {
116+
r := newHashRange(tt.head, tt.chunks)
117+
118+
var (
119+
starts = []common.Hash{{}}
120+
ends = []common.Hash{r.End()}
121+
)
122+
for r.Next() {
123+
starts = append(starts, r.Start())
124+
ends = append(ends, r.End())
125+
}
126+
if len(starts) != len(tt.starts) {
127+
t.Errorf("test %d: starts count mismatch: have %d, want %d", i, len(starts), len(tt.starts))
128+
}
129+
for j := 0; j < len(starts) && j < len(tt.starts); j++ {
130+
if starts[j] != tt.starts[j] {
131+
t.Errorf("test %d, start %d: hash mismatch: have %x, want %x", i, j, starts[j], tt.starts[j])
132+
}
133+
}
134+
if len(ends) != len(tt.ends) {
135+
t.Errorf("test %d: ends count mismatch: have %d, want %d", i, len(ends), len(tt.ends))
136+
}
137+
for j := 0; j < len(ends) && j < len(tt.ends); j++ {
138+
if ends[j] != tt.ends[j] {
139+
t.Errorf("test %d, end %d: hash mismatch: have %x, want %x", i, j, ends[j], tt.ends[j])
140+
}
141+
}
142+
}
143+
}

0 commit comments

Comments
 (0)