Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion client/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ require (
github.com/opentracing/opentracing-go v1.2.0
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
github.com/prometheus/client_golang v1.20.5
github.com/stretchr/testify v1.9.0
Expand Down
4 changes: 2 additions & 2 deletions client/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 h1:OcZxUJEwZzFIqY8AkRIHuEK8U1X5OyLfqAwVnhaKsag=
github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 h1:ENZsbCKjZe5GyNVuKHTGYiYd4LnM8PRphBsYU6M19g0=
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ require (
github.com/pingcap/errcode v0.3.0
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
github.com/pingcap/metering_sdk v0.0.0-20250918015914-468cd6feb1dc
github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -469,8 +469,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599 h1:57fBeBND/j/dp7nVlw+cWEwYlt4u8CAe4ApsmAEb1ow=
github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 h1:ENZsbCKjZe5GyNVuKHTGYiYd4LnM8PRphBsYU6M19g0=
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
Expand Down
27 changes: 23 additions & 4 deletions pkg/core/region.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,14 @@ type RegionInfo struct {
queryStats *pdpb.QueryStats
flowRoundDivisor uint64
// buckets is not thread unsafe, it should be accessed by the request `report buckets` with greater version.
// todo: keep it compatible with previous design, we can remove it later.
buckets unsafe.Pointer
// source is used to indicate region's source, such as Storage/Sync/Heartbeat.
source RegionSource
// ref is used to indicate the reference count of the region in root-tree and sub-tree.
ref atomic.Int32
// bucketMeta is used to store the bucket meta reported by tikv.
bucketMeta *metapb.BucketMeta
}

// RegionSource is the source of region.
Expand Down Expand Up @@ -215,6 +218,7 @@ type RegionHeartbeatRequest interface {
GetQueryStats() *pdpb.QueryStats
GetApproximateSize() uint64
GetApproximateKeys() uint64
GetBucketMeta() *metapb.BucketMeta
}

// RegionFromHeartbeat constructs a Region from region heartbeat.
Expand Down Expand Up @@ -243,6 +247,7 @@ func RegionFromHeartbeat(heartbeat RegionHeartbeatRequest, flowRoundDivisor uint
queryStats: heartbeat.GetQueryStats(),
source: Heartbeat,
flowRoundDivisor: flowRoundDivisor,
bucketMeta: heartbeat.GetBucketMeta(),
}

// scheduling service doesn't need the following fields.
Expand Down Expand Up @@ -281,7 +286,8 @@ func (r *RegionInfo) Inherit(origin *RegionInfo, bucketEnable bool) {
r.approximateSize = EmptyRegionApproximateSize
}
}
if bucketEnable && origin != nil && r.buckets == nil {
// skip bucket meta update if tikv has report bucket meta.
if r.bucketMeta == nil && bucketEnable && origin != nil && r.buckets == nil {
r.buckets = origin.buckets
}
}
Expand Down Expand Up @@ -316,6 +322,7 @@ func (r *RegionInfo) Clone(opts ...RegionCreateOption) *RegionInfo {
replicationStatus: r.replicationStatus,
buckets: r.buckets,
queryStats: typeutil.DeepClone(r.queryStats, QueryStatsFactory),
bucketMeta: r.bucketMeta,
}

for _, opt := range opts {
Expand Down Expand Up @@ -607,7 +614,7 @@ func (r *RegionInfo) GetStat() *pdpb.RegionStat {
}
}

// UpdateBuckets sets the buckets of the region.
// UpdateBuckets sets the buckets of the region, used by bucket report.
func (r *RegionInfo) UpdateBuckets(buckets, old *metapb.Buckets) bool {
if buckets == nil {
atomic.StorePointer(&r.buckets, nil)
Expand All @@ -627,6 +634,13 @@ func (r *RegionInfo) GetBuckets() *metapb.Buckets {
if r == nil {
return nil
}
if meta := r.bucketMeta; meta != nil {
return &metapb.Buckets{
RegionId: r.GetID(),
Version: meta.GetVersion(),
Keys: meta.GetKeys(),
}
}
buckets := atomic.LoadPointer(&r.buckets)
return (*metapb.Buckets)(buckets)
}
Expand Down Expand Up @@ -885,9 +899,14 @@ func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc {
saveKV, saveCache = true, true
return
}
if len(region.GetBuckets().GetKeys()) != len(origin.GetBuckets().GetKeys()) {
// If bucket version increased, will update both kv and cache.
// If bucket version is 0 but previously is not, which means tikv has disabled bucket, will also update both kv and cache to be compatible.
if region.GetBuckets().GetVersion() > origin.GetBuckets().GetVersion() || (region.GetBuckets().GetVersion() == 0 && origin.GetBuckets().GetVersion() > 0) {
if log.GetLevel() <= zap.DebugLevel {
debug("bucket key changed", zap.Uint64("region-id", region.GetID()))
debug("bucket key changed",
zap.Uint64("region-id", region.GetID()),
zap.Uint64("old-bucket-version", origin.GetBuckets().GetVersion()),
zap.Uint64("new-bucket-version", region.GetBuckets().GetVersion()))
}
saveKV, saveCache = true, true
return
Expand Down
9 changes: 8 additions & 1 deletion pkg/core/region_option.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,13 @@ func WithIncVersion() RegionCreateOption {
}
}

// WithBucketMeta sets the bucket meta for the region.
func WithBucketMeta(bucketMeta *metapb.BucketMeta) RegionCreateOption {
return func(region *RegionInfo) {
region.bucketMeta = bucketMeta
}
}

// WithDecVersion decreases the version of the region.
func WithDecVersion() RegionCreateOption {
return func(region *RegionInfo) {
Expand Down Expand Up @@ -230,7 +237,7 @@ func WithRemoveStorePeer(storeID uint64) RegionCreateOption {
}
}

// SetBuckets sets the buckets for the region, only use test.
// SetBuckets sets the buckets for the region, only use test and region syncer, it will update bucket meta also.
func SetBuckets(buckets *metapb.Buckets) RegionCreateOption {
return func(region *RegionInfo) {
region.UpdateBuckets(buckets, region.GetBuckets())
Expand Down
32 changes: 32 additions & 0 deletions pkg/core/region_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1428,3 +1428,35 @@ func TestResetRegionCache(t *testing.T) {
re.Equal(1, regions.GetTotalRegionCount())
re.NotNil(regions.GetRegion(4))
}

func TestGetBucketMeta(t *testing.T) {
re := require.New(t)
region := NewTestRegionInfo(1, 1, []byte("a"), []byte("d"))
re.Nil(region.GetBuckets())
origin := NewTestRegionInfo(1, 2, []byte("a"), []byte("d"))
bucket := &metapb.Buckets{
RegionId: 100,
Version: 1,
Keys: [][]byte{[]byte("a"), []byte("b"), []byte("d")},
}
origin.UpdateBuckets(bucket, nil)
re.Equal(uint64(1), origin.GetBuckets().GetVersion())
region.Inherit(origin, true)
re.Equal(uint64(1), region.GetBuckets().GetVersion())

// Inherit false if region has bucket meta
bucket1 := &metapb.Buckets{
RegionId: 100,
Version: 2,
Keys: [][]byte{[]byte("a"), []byte("b"), []byte("d")},
}
re.True(origin.UpdateBuckets(bucket1, origin.GetBuckets()))
re.Equal(uint64(2), origin.GetBuckets().GetVersion())
region.bucketMeta = &metapb.BucketMeta{
Version: 1,
Keys: [][]byte{[]byte("a"), []byte("b"), []byte("d")},
}
region.Inherit(origin, true)
// Inherit should not override buckets when region already has bucketMeta
re.Equal(uint64(1), region.GetBuckets().GetVersion())
}
5 changes: 5 additions & 0 deletions pkg/syncer/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,11 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) {
if hasBuckets {
if old := origin.GetBuckets(); buckets[i].GetVersion() > old.GetVersion() {
region.UpdateBuckets(buckets[i], old)
meta := &metapb.BucketMeta{
Version: buckets[i].GetVersion(),
Keys: buckets[i].GetKeys(),
}
region = region.Clone(core.WithBucketMeta(meta))
}
}
if saveKV {
Expand Down
36 changes: 36 additions & 0 deletions server/cluster/cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -617,6 +617,42 @@ func TestStoreClusterVersion(t *testing.T) {
re.Equal(s1.Version, cluster.GetClusterVersion())
}

func TestStaleBucketMeta(t *testing.T) {
re := require.New(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

_, opt, err := newTestScheduleConfig()
re.NoError(err)
cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend())
cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil)

region := core.NewTestRegionInfo(1, 1, []byte{'a'}, []byte{'d'})
bucket1 := &metapb.Buckets{
RegionId: 1,
Version: 2,
Keys: [][]byte{{'a'}, {'d'}},
}
re.True(region.UpdateBuckets(bucket1, nil))
re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region))
re.Equal(bucket1, cluster.GetRegion(1).GetBuckets())

// region split
newRegion := region.Clone(core.WithIncVersion(), core.WithStartKey([]byte{'c'}), core.WithEndKey([]byte{'d'}))
re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), newRegion))
re.Equal(bucket1, cluster.GetRegion(1).GetBuckets())

// region heartbeat with bucket meta
bucket2 := &metapb.BucketMeta{
Version: 3,
Keys: [][]byte{{'c'}, {'d'}},
}
region2 := newRegion.Clone(core.WithIncVersion(), core.WithBucketMeta(bucket2))
re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region2))
region1 := cluster.GetRegion(1)
re.NotEqual(bucket1, region1.GetBuckets())
}

func TestRegionHeartbeatHotStat(t *testing.T) {
re := require.New(t)
ctx, cancel := context.WithCancel(context.Background())
Expand Down
2 changes: 1 addition & 1 deletion tests/integrations/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ require (
github.com/go-sql-driver/mysql v1.7.0
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
github.com/prometheus/client_golang v1.20.5
github.com/prometheus/client_model v0.6.1
Expand Down
4 changes: 2 additions & 2 deletions tests/integrations/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -462,8 +462,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599 h1:57fBeBND/j/dp7nVlw+cWEwYlt4u8CAe4ApsmAEb1ow=
github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 h1:ENZsbCKjZe5GyNVuKHTGYiYd4LnM8PRphBsYU6M19g0=
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
Expand Down
2 changes: 1 addition & 1 deletion tools/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ require (
github.com/mattn/go-shellwords v1.0.12
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
github.com/pmezard/go-difflib v1.0.0
github.com/prometheus/client_golang v1.20.5
Expand Down
4 changes: 2 additions & 2 deletions tools/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599 h1:57fBeBND/j/dp7nVlw+cWEwYlt4u8CAe4ApsmAEb1ow=
github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 h1:ENZsbCKjZe5GyNVuKHTGYiYd4LnM8PRphBsYU6M19g0=
github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
Expand Down