Skip to content

Commit d85f796

Browse files
zsfelfoldifjl
zsfelfoldi
and
fjl
authored
eth/filters: implement log filter using new log index (#31080)
This PR is #2 of a 3-part series that implements the new log index intended to replace core/bloombits. Based on #31079 Replaces #30370 This part replaces the old bloombits based log search logic in `eth/filters` to use the new `core/filtermaps` logic. FilterMaps data structure explanation: https://gist.github.com/zsfelfoldi/a60795f9da7ae6422f28c7a34e02a07e Log index generator code overview: https://gist.github.com/zsfelfoldi/97105dff0b1a4f5ed557924a24b9b9e7 Search pattern matcher code overview: https://gist.github.com/zsfelfoldi/5981735641c956afb18065e84f8aff34 Note that the possibility of a tree hashing scheme and remote proof protocol are mentioned in the documents above but they are not exactly specified yet. These specs are WIP and will be finalized after the local log indexer/filter code is finalized and merged. --------- Co-authored-by: Felix Lange <[email protected]>
1 parent 0f06e35 commit d85f796

25 files changed

+1585
-1006
lines changed

‎cmd/geth/chaincmd.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ if one is set. Otherwise it prints the genesis from the datadir.`,
100100
utils.VMTraceFlag,
101101
utils.VMTraceJsonConfigFlag,
102102
utils.TransactionHistoryFlag,
103+
utils.LogHistoryFlag,
104+
utils.LogNoHistoryFlag,
105+
utils.LogExportCheckpointsFlag,
103106
utils.StateHistoryFlag,
104107
}, utils.DatabaseFlags),
105108
Description: `

‎cmd/geth/main.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ var (
8787
utils.TxLookupLimitFlag, // deprecated
8888
utils.TransactionHistoryFlag,
8989
utils.ChainHistoryFlag,
90+
utils.LogHistoryFlag,
91+
utils.LogNoHistoryFlag,
92+
utils.LogExportCheckpointsFlag,
9093
utils.StateHistoryFlag,
9194
utils.LightServeFlag, // deprecated
9295
utils.LightIngressFlag, // deprecated

‎cmd/utils/flags.go

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,23 @@ var (
278278
Value: ethconfig.Defaults.HistoryMode.String(),
279279
Category: flags.StateCategory,
280280
}
281+
LogHistoryFlag = &cli.Uint64Flag{
282+
Name: "history.logs",
283+
Usage: "Number of recent blocks to maintain log search index for (default = about one year, 0 = entire chain)",
284+
Value: ethconfig.Defaults.LogHistory,
285+
Category: flags.StateCategory,
286+
}
287+
LogNoHistoryFlag = &cli.BoolFlag{
288+
Name: "history.logs.disable",
289+
Usage: "Do not maintain log search index",
290+
Category: flags.StateCategory,
291+
}
292+
LogExportCheckpointsFlag = &cli.StringFlag{
293+
Name: "history.logs.export",
294+
Usage: "Export checkpoints to file in go source file format",
295+
Category: flags.StateCategory,
296+
Value: "",
297+
}
281298
// Beacon client light sync settings
282299
BeaconApiFlag = &cli.StringSliceFlag{
283300
Name: "beacon.api",
@@ -1636,6 +1653,15 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
16361653
cfg.StateScheme = rawdb.HashScheme
16371654
log.Warn("Forcing hash state-scheme for archive mode")
16381655
}
1656+
if ctx.IsSet(LogHistoryFlag.Name) {
1657+
cfg.LogHistory = ctx.Uint64(LogHistoryFlag.Name)
1658+
}
1659+
if ctx.IsSet(LogNoHistoryFlag.Name) {
1660+
cfg.LogNoHistory = true
1661+
}
1662+
if ctx.IsSet(LogExportCheckpointsFlag.Name) {
1663+
cfg.LogExportCheckpoints = ctx.String(LogExportCheckpointsFlag.Name)
1664+
}
16391665
if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) {
16401666
cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100
16411667
}

‎common/range.go

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
// Copyright 2025 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package common
18+
19+
import (
20+
"iter"
21+
)
22+
23+
// Range represents a range of integers.
24+
type Range[T uint32 | uint64] struct {
25+
first, afterLast T
26+
}
27+
28+
// NewRange creates a new range based of first element and number of elements.
29+
func NewRange[T uint32 | uint64](first, count T) Range[T] {
30+
return Range[T]{first, first + count}
31+
}
32+
33+
// First returns the first element of the range.
34+
func (r Range[T]) First() T {
35+
return r.first
36+
}
37+
38+
// Last returns the last element of the range. This panics for empty ranges.
39+
func (r Range[T]) Last() T {
40+
if r.first == r.afterLast {
41+
panic("last item of zero length range is not allowed")
42+
}
43+
return r.afterLast - 1
44+
}
45+
46+
// AfterLast returns the first element after the range. This allows obtaining
47+
// information about the end part of zero length ranges.
48+
func (r Range[T]) AfterLast() T {
49+
return r.afterLast
50+
}
51+
52+
// Count returns the number of elements in the range.
53+
func (r Range[T]) Count() T {
54+
return r.afterLast - r.first
55+
}
56+
57+
// IsEmpty returns true if the range is empty.
58+
func (r Range[T]) IsEmpty() bool {
59+
return r.first == r.afterLast
60+
}
61+
62+
// Includes returns true if the given element is inside the range.
63+
func (r Range[T]) Includes(v T) bool {
64+
return v >= r.first && v < r.afterLast
65+
}
66+
67+
// SetFirst updates the first element of the list.
68+
func (r *Range[T]) SetFirst(v T) {
69+
r.first = v
70+
if r.afterLast < r.first {
71+
r.afterLast = r.first
72+
}
73+
}
74+
75+
// SetAfterLast updates the end of the range by specifying the first element
76+
// after the range. This allows setting zero length ranges.
77+
func (r *Range[T]) SetAfterLast(v T) {
78+
r.afterLast = v
79+
if r.afterLast < r.first {
80+
r.first = r.afterLast
81+
}
82+
}
83+
84+
// SetLast updates last element of the range.
85+
func (r *Range[T]) SetLast(v T) {
86+
r.SetAfterLast(v + 1)
87+
}
88+
89+
// Intersection returns the intersection of two ranges.
90+
func (r Range[T]) Intersection(q Range[T]) Range[T] {
91+
i := Range[T]{first: max(r.first, q.first), afterLast: min(r.afterLast, q.afterLast)}
92+
if i.first > i.afterLast {
93+
return Range[T]{}
94+
}
95+
return i
96+
}
97+
98+
// Union returns the union of two ranges. Panics for gapped ranges.
99+
func (r Range[T]) Union(q Range[T]) Range[T] {
100+
if max(r.first, q.first) > min(r.afterLast, q.afterLast) {
101+
panic("cannot create union; gap between ranges")
102+
}
103+
return Range[T]{first: min(r.first, q.first), afterLast: max(r.afterLast, q.afterLast)}
104+
}
105+
106+
// Iter iterates all integers in the range.
107+
func (r Range[T]) Iter() iter.Seq[T] {
108+
return func(yield func(T) bool) {
109+
for i := r.first; i < r.afterLast; i++ {
110+
if !yield(i) {
111+
break
112+
}
113+
}
114+
}
115+
}

‎common/range_test.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Copyright 2025 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package common
18+
19+
import (
20+
"slices"
21+
"testing"
22+
)
23+
24+
func TestRangeIter(t *testing.T) {
25+
r := NewRange[uint32](1, 7)
26+
values := slices.Collect(r.Iter())
27+
if !slices.Equal(values, []uint32{1, 2, 3, 4, 5, 6, 7}) {
28+
t.Fatalf("wrong iter values: %v", values)
29+
}
30+
31+
empty := NewRange[uint32](1, 0)
32+
values = slices.Collect(empty.Iter())
33+
if !slices.Equal(values, []uint32{}) {
34+
t.Fatalf("wrong iter values: %v", values)
35+
}
36+
}

‎core/filtermaps/checkpoints_holesky.json

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
11
[
2-
{"blockNumber": 814411, "blockId": "0xf763e96fc3920359c5f706803024b78e83796a3a8563bb5a83c3ddd7cbfde287", "firstIndex": 67107637},
3-
{"blockNumber": 914278, "blockId": "0x0678cf8d53c0d6d27896df657d98cc73bc63ca468b6295068003938ef9b0f927", "firstIndex": 134217671},
4-
{"blockNumber": 1048874, "blockId": "0x3620c3d52a40ff4d9fc58c3104cfa2f327f55592caf6a2394c207a5e00b4f740", "firstIndex": 201326382},
5-
{"blockNumber": 1144441, "blockId": "0x438fb42850f5a0d8e1666de598a4d0106b62da0f7448c62fe029b8cbad35d08d", "firstIndex": 268435440},
6-
{"blockNumber": 1230411, "blockId": "0xf0ee07e60a93910723b259473a253dd9cf674e8b78c4f153b32ad7032efffeeb", "firstIndex": 335543079},
7-
{"blockNumber": 1309112, "blockId": "0xc1646e5ef4b4343880a85b1a4111e3321d609a1225e9cebbe10d1c7abf99e58d", "firstIndex": 402653100},
8-
{"blockNumber": 1380522, "blockId": "0x1617cae91989d97ac6335c4217aa6cc7f7f4c2837e20b3b5211d98d6f9e97e44", "firstIndex": 469761917},
9-
{"blockNumber": 1476962, "blockId": "0xd978455d2618d093dfc685d7f43f61be6dae0fa8a9cb915ae459aa6e0a5525f0", "firstIndex": 536870773},
10-
{"blockNumber": 1533518, "blockId": "0xe7d39d71bd9d5f1f3157c35e0329531a7950a19e3042407e38948b89b5384f78", "firstIndex": 603979664},
11-
{"blockNumber": 1613787, "blockId": "0xa793168d135c075732a618ec367faaed5f359ffa81898c73cb4ec54ec2caa696", "firstIndex": 671088003},
12-
{"blockNumber": 1719099, "blockId": "0xc4394c71a8a24efe64c5ff2afcdd1594f3708524e6084aa7dadd862bd704ab03", "firstIndex": 738196914},
13-
{"blockNumber": 1973165, "blockId": "0xee3a9e959a437c707a3036736ec8d42a9261ac6100972c26f65eedcde315a81d", "firstIndex": 805306333},
14-
{"blockNumber": 2274844, "blockId": "0x76e2d33653ed9282c63ad09d721e1f2e29064aa9c26202e20fc4cc73e8dfe5f6", "firstIndex": 872415141},
15-
{"blockNumber": 2530503, "blockId": "0x59f4e45345f8b8f848be5004fe75c4a28f651864256c3aa9b2da63369432b718", "firstIndex": 939523693},
16-
{"blockNumber": 2781903, "blockId": "0xc981e91c6fb69c5e8146ead738fcfc561831f11d7786d39c7fa533966fc37675", "firstIndex": 1006632906},
17-
{"blockNumber": 3101713, "blockId": "0xc7baa577c91d8439e3fc79002d2113d07ca54a4724bf2f1f5af937b7ba8e1f32", "firstIndex": 1073741382},
18-
{"blockNumber": 3221770, "blockId": "0xa6b8240b7883fcc71aa5001b5ba66c889975c5217e14c16edebdd6f6e23a9424", "firstIndex": 1140850360}
2+
{"blockNumber": 814410, "blockId": "0x6c38f0d4ff2c23ae187f581cebb0963c0ec2dbf051b289de1582c369c98a3244", "firstIndex": 67107349},
3+
{"blockNumber": 914268, "blockId": "0xeff161573a11eb6e2ab930674a5245b2c5ffc5e9e63093503344ec6fa60578a3", "firstIndex": 134216064},
4+
{"blockNumber": 1048866, "blockId": "0xebd3b95415dad9ab7f2b1d25e48c791e299063c09427bbde54217029c3e215ad", "firstIndex": 201325914},
5+
{"blockNumber": 1144433, "blockId": "0x0c895438ef12a2c835b4372c209e40a499ba2b18ed0e658846813784de391392", "firstIndex": 268434935},
6+
{"blockNumber": 1230406, "blockId": "0xd3512e7241efc9853e46b1ad565403d302f62457b6dd84917c31ff375fabf487", "firstIndex": 335544096},
7+
{"blockNumber": 1309104, "blockId": "0xcf108c6e002e7a5657bf7587adde03edf5f20d03e95b66a194eb8080daaf4f97", "firstIndex": 402653143},
8+
{"blockNumber": 1380499, "blockId": "0x984b0f8b6bf06f1240bbca8c1df9bef3880bedafd5b5c2a55cbc2f64c9efb974", "firstIndex": 469759822},
9+
{"blockNumber": 1476950, "blockId": "0x8323b528bb8d80a96b172de92452be0f45078a9ca311cb4be6675f089e25a426", "firstIndex": 536870335},
10+
{"blockNumber": 1533506, "blockId": "0x737dc416070aa3b522a0c2ab813a70c1820f062c718f27597394f309f0004106", "firstIndex": 603979618},
11+
{"blockNumber": 1613764, "blockId": "0x9d6a5a505afdda86b1ebcc2b10cb687a1f89c2e2b74315945a3ddc6a0b3c9cd6", "firstIndex": 671088253},
12+
{"blockNumber": 1719073, "blockId": "0x17bf6a51d9c55a908c3e9e652f80b2aa464b049c697abf3bd5a537e461aff0b9", "firstIndex": 738197366},
13+
{"blockNumber": 1973133, "blockId": "0x14b288d70e688de3334d09283670301aacfed21c193da8a56fd63767f8177705", "firstIndex": 805305624},
14+
{"blockNumber": 2274761, "blockId": "0xf02790b273b04219e001d2fcc93e8e47708cb228364e96ad754bc8050d08a9aa", "firstIndex": 872414871},
15+
{"blockNumber": 2530470, "blockId": "0x157ea98b1a7e66dd3f045da8e25219800671f9a000211d3d94006efd33d89a80", "firstIndex": 939523234},
16+
{"blockNumber": 2781706, "blockId": "0xa723663a584e280700d2302eba03d1b3b6549333db70c6152576d33e2911f594", "firstIndex": 1006632936},
17+
{"blockNumber": 3101669, "blockId": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353},
18+
{"blockNumber": 3221725, "blockId": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137},
19+
{"blockNumber": 3357164, "blockId": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453},
20+
{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542}
1921
]
20-

0 commit comments

Comments
 (0)