diff --git a/client/go.mod b/client/go.mod index fffd579e02f..c49122981fc 100644 --- a/client/go.mod +++ b/client/go.mod @@ -10,7 +10,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 + github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 github.com/stretchr/testify v1.9.0 diff --git a/client/go.sum b/client/go.sum index 65c46e129d0..c70d8f6e295 100644 --- a/client/go.sum +++ b/client/go.sum @@ -49,8 +49,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= -github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 h1:OcZxUJEwZzFIqY8AkRIHuEK8U1X5OyLfqAwVnhaKsag= -github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 h1:ENZsbCKjZe5GyNVuKHTGYiYd4LnM8PRphBsYU6M19g0= +github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/go.mod b/go.mod index 658eab56489..0754133015f 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599 + github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/metering_sdk v0.0.0-20250918015914-468cd6feb1dc github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 diff --git a/go.sum b/go.sum index f3c96a99fec..642446f4184 100644 --- a/go.sum +++ b/go.sum @@ -469,8 +469,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599 h1:57fBeBND/j/dp7nVlw+cWEwYlt4u8CAe4ApsmAEb1ow= -github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 h1:ENZsbCKjZe5GyNVuKHTGYiYd4LnM8PRphBsYU6M19g0= +github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/pkg/core/region.go b/pkg/core/region.go index c848d4670ce..eb431cbdc69 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -85,11 +85,14 @@ type RegionInfo struct { queryStats *pdpb.QueryStats flowRoundDivisor uint64 // buckets is not thread unsafe, it should be accessed by the request `report buckets` with greater version. + // todo: keep it compatible with previous design, we can remove it later. buckets unsafe.Pointer // source is used to indicate region's source, such as Storage/Sync/Heartbeat. source RegionSource // ref is used to indicate the reference count of the region in root-tree and sub-tree. ref atomic.Int32 + // bucketMeta is used to store the bucket meta reported by tikv. + bucketMeta *metapb.BucketMeta } // RegionSource is the source of region. @@ -215,6 +218,7 @@ type RegionHeartbeatRequest interface { GetQueryStats() *pdpb.QueryStats GetApproximateSize() uint64 GetApproximateKeys() uint64 + GetBucketMeta() *metapb.BucketMeta } // RegionFromHeartbeat constructs a Region from region heartbeat. @@ -243,6 +247,7 @@ func RegionFromHeartbeat(heartbeat RegionHeartbeatRequest, flowRoundDivisor uint queryStats: heartbeat.GetQueryStats(), source: Heartbeat, flowRoundDivisor: flowRoundDivisor, + bucketMeta: heartbeat.GetBucketMeta(), } // scheduling service doesn't need the following fields. @@ -281,7 +286,8 @@ func (r *RegionInfo) Inherit(origin *RegionInfo, bucketEnable bool) { r.approximateSize = EmptyRegionApproximateSize } } - if bucketEnable && origin != nil && r.buckets == nil { + // skip bucket meta update if tikv has report bucket meta. + if r.bucketMeta == nil && bucketEnable && origin != nil && r.buckets == nil { r.buckets = origin.buckets } } @@ -316,6 +322,7 @@ func (r *RegionInfo) Clone(opts ...RegionCreateOption) *RegionInfo { replicationStatus: r.replicationStatus, buckets: r.buckets, queryStats: typeutil.DeepClone(r.queryStats, QueryStatsFactory), + bucketMeta: r.bucketMeta, } for _, opt := range opts { @@ -607,7 +614,7 @@ func (r *RegionInfo) GetStat() *pdpb.RegionStat { } } -// UpdateBuckets sets the buckets of the region. +// UpdateBuckets sets the buckets of the region, used by bucket report. func (r *RegionInfo) UpdateBuckets(buckets, old *metapb.Buckets) bool { if buckets == nil { atomic.StorePointer(&r.buckets, nil) @@ -627,6 +634,13 @@ func (r *RegionInfo) GetBuckets() *metapb.Buckets { if r == nil { return nil } + if meta := r.bucketMeta; meta != nil { + return &metapb.Buckets{ + RegionId: r.GetID(), + Version: meta.GetVersion(), + Keys: meta.GetKeys(), + } + } buckets := atomic.LoadPointer(&r.buckets) return (*metapb.Buckets)(buckets) } @@ -885,9 +899,14 @@ func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc { saveKV, saveCache = true, true return } - if len(region.GetBuckets().GetKeys()) != len(origin.GetBuckets().GetKeys()) { + // If bucket version increased, will update both kv and cache. + // If bucket version is 0 but previously is not, which means tikv has disabled bucket, will also update both kv and cache to be compatible. + if region.GetBuckets().GetVersion() > origin.GetBuckets().GetVersion() || (region.GetBuckets().GetVersion() == 0 && origin.GetBuckets().GetVersion() > 0) { if log.GetLevel() <= zap.DebugLevel { - debug("bucket key changed", zap.Uint64("region-id", region.GetID())) + debug("bucket key changed", + zap.Uint64("region-id", region.GetID()), + zap.Uint64("old-bucket-version", origin.GetBuckets().GetVersion()), + zap.Uint64("new-bucket-version", region.GetBuckets().GetVersion())) } saveKV, saveCache = true, true return diff --git a/pkg/core/region_option.go b/pkg/core/region_option.go index a726d6d8669..587ed3c4e44 100644 --- a/pkg/core/region_option.go +++ b/pkg/core/region_option.go @@ -153,6 +153,13 @@ func WithIncVersion() RegionCreateOption { } } +// WithBucketMeta sets the bucket meta for the region. +func WithBucketMeta(bucketMeta *metapb.BucketMeta) RegionCreateOption { + return func(region *RegionInfo) { + region.bucketMeta = bucketMeta + } +} + // WithDecVersion decreases the version of the region. func WithDecVersion() RegionCreateOption { return func(region *RegionInfo) { @@ -230,7 +237,7 @@ func WithRemoveStorePeer(storeID uint64) RegionCreateOption { } } -// SetBuckets sets the buckets for the region, only use test. +// SetBuckets sets the buckets for the region, only use test and region syncer. func SetBuckets(buckets *metapb.Buckets) RegionCreateOption { return func(region *RegionInfo) { region.UpdateBuckets(buckets, region.GetBuckets()) diff --git a/pkg/core/region_test.go b/pkg/core/region_test.go index 80e46c90135..799c7fb0bb0 100644 --- a/pkg/core/region_test.go +++ b/pkg/core/region_test.go @@ -1428,3 +1428,35 @@ func TestResetRegionCache(t *testing.T) { re.Equal(1, regions.GetTotalRegionCount()) re.NotNil(regions.GetRegion(4)) } + +func TestGetBucketMeta(t *testing.T) { + re := require.New(t) + region := NewTestRegionInfo(1, 1, []byte("a"), []byte("d")) + re.Nil(region.GetBuckets()) + origin := NewTestRegionInfo(1, 2, []byte("a"), []byte("d")) + bucket := &metapb.Buckets{ + RegionId: 100, + Version: 1, + Keys: [][]byte{[]byte("a"), []byte("b"), []byte("d")}, + } + origin.UpdateBuckets(bucket, nil) + re.Equal(uint64(1), origin.GetBuckets().GetVersion()) + region.Inherit(origin, true) + re.Equal(uint64(1), region.GetBuckets().GetVersion()) + + // Inherit false if region has bucket meta + bucket1 := &metapb.Buckets{ + RegionId: 100, + Version: 2, + Keys: [][]byte{[]byte("a"), []byte("b"), []byte("d")}, + } + re.True(origin.UpdateBuckets(bucket1, origin.GetBuckets())) + re.Equal(uint64(2), origin.GetBuckets().GetVersion()) + region.bucketMeta = &metapb.BucketMeta{ + Version: 1, + Keys: [][]byte{[]byte("a"), []byte("b"), []byte("d")}, + } + region.Inherit(origin, true) + // Inherit should not override buckets when region already has bucketMeta + re.Equal(uint64(1), region.GetBuckets().GetVersion()) +} diff --git a/pkg/syncer/client.go b/pkg/syncer/client.go index 8808bd5ce87..4c02058ef1f 100644 --- a/pkg/syncer/client.go +++ b/pkg/syncer/client.go @@ -236,13 +236,18 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) { // no limit for followers. } saveKV, _, _, _ := regionGuide(cctx, region, origin) - overlaps := bc.PutRegion(region) if hasBuckets { if old := origin.GetBuckets(); buckets[i].GetVersion() > old.GetVersion() { region.UpdateBuckets(buckets[i], old) + meta := &metapb.BucketMeta{ + Version: buckets[i].GetVersion(), + Keys: buckets[i].GetKeys(), + } + region = region.Clone(core.WithBucketMeta(meta)) } } + overlaps := bc.PutRegion(region) if saveKV { err = regionStorage.SaveRegion(r) } diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 7f4fc11b159..9d6a7552f17 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -617,6 +617,42 @@ func TestStoreClusterVersion(t *testing.T) { re.Equal(s1.Version, cluster.GetClusterVersion()) } +func TestStaleBucketMeta(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, opt, err := newTestScheduleConfig() + re.NoError(err) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) + + region := core.NewTestRegionInfo(1, 1, []byte{'a'}, []byte{'d'}) + bucket1 := &metapb.Buckets{ + RegionId: 1, + Version: 2, + Keys: [][]byte{{'a'}, {'d'}}, + } + re.True(region.UpdateBuckets(bucket1, nil)) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) + re.Equal(bucket1, cluster.GetRegion(1).GetBuckets()) + + // region split + newRegion := region.Clone(core.WithIncVersion(), core.WithStartKey([]byte{'c'}), core.WithEndKey([]byte{'d'})) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), newRegion)) + re.Equal(bucket1, cluster.GetRegion(1).GetBuckets()) + + // region heartbeat with bucket meta + bucket2 := &metapb.BucketMeta{ + Version: 3, + Keys: [][]byte{{'c'}, {'d'}}, + } + region2 := newRegion.Clone(core.WithIncVersion(), core.WithBucketMeta(bucket2)) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region2)) + region1 := cluster.GetRegion(1) + re.NotEqual(bucket1, region1.GetBuckets()) +} + func TestRegionHeartbeatHotStat(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index beecbfb4ad2..413a728c2f0 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -14,7 +14,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599 + github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index 62634117da7..b3320ba9b32 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -462,8 +462,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599 h1:57fBeBND/j/dp7nVlw+cWEwYlt4u8CAe4ApsmAEb1ow= -github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 h1:ENZsbCKjZe5GyNVuKHTGYiYd4LnM8PRphBsYU6M19g0= +github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/tools/go.mod b/tools/go.mod index 641680e6d19..b17ea58bca6 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -23,7 +23,7 @@ require ( github.com/mattn/go-shellwords v1.0.12 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599 + github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.20.5 diff --git a/tools/go.sum b/tools/go.sum index 30d26f4a890..d5f7ac0f8f5 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -467,8 +467,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599 h1:57fBeBND/j/dp7nVlw+cWEwYlt4u8CAe4ApsmAEb1ow= -github.com/pingcap/kvproto v0.0.0-20250923091925-d79d11002599/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7 h1:ENZsbCKjZe5GyNVuKHTGYiYd4LnM8PRphBsYU6M19g0= +github.com/pingcap/kvproto v0.0.0-20260106110113-438649d89ee7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=