diff --git a/client/client.go b/client/client.go index 3d0c364e817..ab1d32ec137 100644 --- a/client/client.go +++ b/client/client.go @@ -68,14 +68,18 @@ type RPCClient interface { // UpdateGCSafePoint TiKV will check it and do GC themselves if necessary. // If the given safePoint is less than the current one, it will not be updated. // Returns the new safePoint after updating. + // + // Deprecated: This API is deprecated and replaced by AdvanceGCSafePoint, which expected only for use of the + // GCWorker of TiDB or any component that is responsible for managing and driving GC. For callers that want to + // read the current GC safe point, consider using GetGCStates instead. UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint64, error) // UpdateServiceGCSafePoint updates the safepoint for specific service and // returns the minimum safepoint across all services, this value is used to // determine the safepoint for multiple services, it does not trigger a GC // job. Use UpdateGCSafePoint to trigger the GC job if needed. + // + // Deprecated: This API is deprecated and replaced by SetGCBarrier and DeleteGCBarrier. UpdateServiceGCSafePoint(ctx context.Context, serviceID string, ttl int64, safePoint uint64) (uint64, error) - // Deprecated: Avoid using this API. - WatchGCSafePointV2(ctx context.Context, revision int64) (chan []*pdpb.SafePointEvent, error) // ScatterRegion scatters the specified region. Should use it for a batch of regions, // and the distribution of these regions will be dispersed. // NOTICE: This method is the old version of ScatterRegions, you should use the later one as your first choice. @@ -1033,6 +1037,10 @@ func (c *client) GetAllStores(ctx context.Context, opts ...opt.GetStoreOption) ( } // UpdateGCSafePoint implements the RPCClient interface. +// +// Deprecated: This API is deprecated and replaced by AdvanceGCSafePoint, which expected only for use of the +// GCWorker of TiDB or any component that is responsible for managing and driving GC. For callers that want to +// read the current GC safe point, consider using GetGCStates instead. func (c *client) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint64, error) { if c.inner.keyspaceID != constants.NullKeyspaceID { return c.updateGCSafePointV2(ctx, c.inner.keyspaceID, safePoint) @@ -1067,6 +1075,8 @@ func (c *client) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint6 // returns the minimum safepoint across all services, this value is used to // determine the safepoint for multiple services, it does not trigger a GC // job. Use UpdateGCSafePoint to trigger the GC job if needed. +// +// Deprecated: This API is deprecated and replaced by SetGCBarrier and DeleteGCBarrier. func (c *client) UpdateServiceGCSafePoint(ctx context.Context, serviceID string, ttl int64, safePoint uint64) (uint64, error) { if c.inner.keyspaceID != constants.NullKeyspaceID { return c.updateServiceSafePointV2(ctx, c.inner.keyspaceID, serviceID, ttl, safePoint) diff --git a/client/clients/gc/client.go b/client/clients/gc/client.go index 1632a4478fc..5a28078b3bd 100644 --- a/client/clients/gc/client.go +++ b/client/clients/gc/client.go @@ -37,11 +37,41 @@ type Client interface { // //nolint:revive type GCStatesClient interface { - // SetGCBarrier sets (creates or updates) a GC barrier. + // SetGCBarrier sets a GC barrier, which blocks GC from being advanced over the given barrierTS for at most a duration + // specified by ttl. This method either adds a new GC barrier or updates an existing one. Returns the information of the + // new GC barrier. + // + // A GC barrier is uniquely identified by the given barrierID in the keyspace scope for NullKeyspace or keyspaces + // with keyspace-level GC enabled. When this method is called on keyspaces without keyspace-level GC enabled, it will + // be equivalent to calling it on the NullKeyspace. + // + // Once a GC barrier is set, it will block the txn safe point from being advanced over the barrierTS, until the GC + // barrier is expired (defined by ttl) or manually deleted (by calling DeleteGCBarrier). + // + // When this method is called on an existing GC barrier, it updates the barrierTS and ttl of the existing GC barrier and + // the expiration time will become the current time plus the ttl. This means that calling this method on an existing + // GC barrier can extend its lifetime arbitrarily. + // + // Passing non-positive value to ttl is not allowed. Passing `time.Duration(math.MaxInt64)` to ttl indicates that the + // GC barrier should never expire. The ttl might be rounded up, and the actual ttl is guaranteed no less than the + // specified duration. + // + // The barrierID must be non-empty. "gc_worker" is a reserved name and cannot be used as a barrierID. + // + // The given barrierTS must be greater than or equal to the current txn safe point, or an error will be returned. + // + // When this function executes successfully, its result is never nil. SetGCBarrier(ctx context.Context, barrierID string, barrierTS uint64, ttl time.Duration) (*GCBarrierInfo, error) - // DeleteGCBarrier deletes a GC barrier. + // DeleteGCBarrier deletes a GC barrier by the given barrierID. Returns the information of the deleted GC barrier, or + // nil if the barrier does not exist. + // + // When this method is called on a keyspace without keyspace-level GC enabled, it will be equivalent to calling it on + // the NullKeyspace. DeleteGCBarrier(ctx context.Context, barrierID string) (*GCBarrierInfo, error) - // GetGCState gets the current GC state. + // GetGCState returns the GC state of the given keyspace. + // + // When this method is called on a keyspace without keyspace-level GC enabled, it will be equivalent to calling it on + // the NullKeyspace. GetGCState(ctx context.Context) (GCState, error) } @@ -51,24 +81,51 @@ type GCStatesClient interface { // WARNING: This is only for internal use. The only possible place to use this is the `GCWorker` in TiDB, or // other possible components that are responsible for being the center of controlling GC of the cluster. type InternalController interface { - // AdvanceTxnSafePoint tries to advance the transaction safe point to the target value. + // AdvanceTxnSafePoint tries to advance the txn safe point to the given target. + // + // Returns a struct AdvanceTxnSafePointResult, which contains the old txn safe point, the target, and the new + // txn safe point it finally made it to advance to. If there's something blocking the txn safe point from being + // advanced to the given target, it may finally be advanced to a smaller value or remains the previous value, in which + // case the BlockerDescription field of the AdvanceTxnSafePointResult will be set to a non-empty string describing + // the reason. + // + // Txn safe point of a single keyspace should never decrease. If the given target is smaller than the previous value, + // it returns an error. + // + // WARNING: This method is only used to manage the GC procedure, and should never be called by code that doesn't + // have the responsibility to manage GC. It can only be called on NullKeyspace or keyspaces with keyspace level GC + // enabled. AdvanceTxnSafePoint(ctx context.Context, target uint64) (AdvanceTxnSafePointResult, error) - // AdvanceGCSafePoint tries to advance the GC safe point to the target value. + // AdvanceGCSafePoint tries to advance the GC safe point to the given target. If the target is less than the current + // value or greater than the txn safe point, it returns an error. + // + // WARNING: This method is only used to manage the GC procedure, and should never be called by code that doesn't + // have the responsibility to manage GC. It can only be called on NullKeyspace or keyspaces with keyspace level GC + // enabled. AdvanceGCSafePoint(ctx context.Context, target uint64) (AdvanceGCSafePointResult, error) } // AdvanceTxnSafePointResult represents the result of advancing transaction safe point. type AdvanceTxnSafePointResult struct { - OldTxnSafePoint uint64 - Target uint64 - NewTxnSafePoint uint64 + // The old txn safe point before the advancement operation. + OldTxnSafePoint uint64 + // The target to which the current advancement operation tried to advance the txn safe point. It contains the + Target uint64 + // same value as the `target` argument passed to the AdvanceTxnSafePoint method. + NewTxnSafePoint uint64 + // When the txn safe point is blocked and is unable to be advanced to exactly the target, this field will contains + // a non-empty string describing the reason why it is blocked. BlockerDescription string } // AdvanceGCSafePointResult represents the result of advancing GC safe point. type AdvanceGCSafePointResult struct { + // The old GC safe point before the advancement operation. OldGCSafePoint uint64 - Target uint64 + // The target to which the current advancement operation tried to advance the GC safe point. It contains the + // same value as the `target` argument passed to the AdvanceGCSafePoint method. + Target uint64 + // The new GC safe point after the advancement operation. NewGCSafePoint uint64 } @@ -97,7 +154,10 @@ func NewGCBarrierInfo(barrierID string, barrierTS uint64, ttl time.Duration, get } } -// IsExpired checks whether the barrier is expired. +// IsExpired checks whether the barrier is expired by the local time. The check is done by checking the local time. +// Note that the result is unreliable in case there is significant time drift between the client and the PD server. +// As the TTL is round down when returning from the server, this method may give an expired result slightly earlier +// than it actually expires in PD server. func (b *GCBarrierInfo) IsExpired() bool { return b.isExpiredImpl(time.Now()) } @@ -115,6 +175,7 @@ func (b *GCBarrierInfo) isExpiredImpl(now time.Time) bool { // //nolint:revive type GCState struct { + // The ID of the keyspace this GC state belongs to. KeyspaceID uint32 TxnSafePoint uint64 GCSafePoint uint64 diff --git a/client/gc_client.go b/client/gc_client.go index 22447303b79..b9937f7d487 100644 --- a/client/gc_client.go +++ b/client/gc_client.go @@ -21,10 +21,8 @@ import ( "time" "github.com/opentracing/opentracing-go" - "go.uber.org/zap" "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/pingcap/log" "github.com/tikv/pd/client/clients/gc" "github.com/tikv/pd/client/constants" @@ -33,6 +31,7 @@ import ( ) // updateGCSafePointV2 update gc safe point for the given keyspace. +// Only used for handling `UpdateGCSafePoint` in keyspace context, which is a deprecated usage. func (c *client) updateGCSafePointV2(ctx context.Context, keyspaceID uint32, safePoint uint64) (uint64, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.UpdateGCSafePointV2", opentracing.ChildOf(span.Context())) @@ -42,6 +41,7 @@ func (c *client) updateGCSafePointV2(ctx context.Context, keyspaceID uint32, saf defer func() { metrics.CmdDurationUpdateGCSafePointV2.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.inner.option.Timeout) + //nolint:staticcheck req := &pdpb.UpdateGCSafePointV2Request{ Header: c.requestHeader(), KeyspaceId: keyspaceID, @@ -52,7 +52,7 @@ func (c *client) updateGCSafePointV2(ctx context.Context, keyspaceID uint32, saf cancel() return 0, errs.ErrClientGetProtoClient } - resp, err := protoClient.UpdateGCSafePointV2(ctx, req) + resp, err := protoClient.UpdateGCSafePointV2(ctx, req) //nolint:staticcheck cancel() if err = c.respForErr(metrics.CmdFailedDurationUpdateGCSafePointV2, start, err, resp.GetHeader()); err != nil { @@ -62,6 +62,7 @@ func (c *client) updateGCSafePointV2(ctx context.Context, keyspaceID uint32, saf } // updateServiceSafePointV2 update service safe point for the given keyspace. +// Only used for handling `UpdateServiceGCSafePoint` in keyspace context, which is a deprecated usage. func (c *client) updateServiceSafePointV2(ctx context.Context, keyspaceID uint32, serviceID string, ttl int64, safePoint uint64) (uint64, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.UpdateServiceSafePointV2", opentracing.ChildOf(span.Context())) @@ -71,6 +72,7 @@ func (c *client) updateServiceSafePointV2(ctx context.Context, keyspaceID uint32 defer func() { metrics.CmdDurationUpdateServiceSafePointV2.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.inner.option.Timeout) + //nolint:staticcheck req := &pdpb.UpdateServiceSafePointV2Request{ Header: c.requestHeader(), KeyspaceId: keyspaceID, @@ -83,7 +85,7 @@ func (c *client) updateServiceSafePointV2(ctx context.Context, keyspaceID uint32 cancel() return 0, errs.ErrClientGetProtoClient } - resp, err := protoClient.UpdateServiceSafePointV2(ctx, req) + resp, err := protoClient.UpdateServiceSafePointV2(ctx, req) //nolint:staticcheck cancel() if err = c.respForErr(metrics.CmdFailedDurationUpdateServiceSafePointV2, start, err, resp.GetHeader()); err != nil { return 0, err @@ -91,50 +93,6 @@ func (c *client) updateServiceSafePointV2(ctx context.Context, keyspaceID uint32 return resp.GetMinSafePoint(), nil } -// WatchGCSafePointV2 watch gc safe point change. -func (c *client) WatchGCSafePointV2(ctx context.Context, revision int64) (chan []*pdpb.SafePointEvent, error) { - SafePointEventsChan := make(chan []*pdpb.SafePointEvent) - req := &pdpb.WatchGCSafePointV2Request{ - Header: c.requestHeader(), - Revision: revision, - } - - ctx, cancel := context.WithTimeout(ctx, c.inner.option.Timeout) - defer cancel() - protoClient, ctx := c.getClientAndContext(ctx) - if protoClient == nil { - return nil, errs.ErrClientGetProtoClient - } - stream, err := protoClient.WatchGCSafePointV2(ctx, req) - if err != nil { - close(SafePointEventsChan) - return nil, err - } - go func() { - defer func() { - close(SafePointEventsChan) - if r := recover(); r != nil { - log.Error("[pd] panic in gc client `WatchGCSafePointV2`", zap.Any("error", r)) - return - } - }() - for { - select { - case <-ctx.Done(): - return - default: - resp, err := stream.Recv() - if err != nil { - log.Error("watch gc safe point v2 error", errs.ZapError(errs.ErrClientWatchGCSafePointV2Stream, err)) - return - } - SafePointEventsChan <- resp.GetEvents() - } - } - }() - return SafePointEventsChan, err -} - // gcInternalController is a stateless wrapper over the client and implements gc.InternalController interface. type gcInternalController struct { client *client diff --git a/pkg/gc/gc_state_manager.go b/pkg/gc/gc_state_manager.go index 3ad7116c5b6..7b994f18c18 100644 --- a/pkg/gc/gc_state_manager.go +++ b/pkg/gc/gc_state_manager.go @@ -148,8 +148,8 @@ func (m *GCStateManager) redirectKeyspace(keyspaceID uint32, isUserAPI bool) (ui } // CompatibleLoadGCSafePoint loads current GC safe point from storage for the legacy GC API `GetGCSafePoint`. -func (m *GCStateManager) CompatibleLoadGCSafePoint() (uint64, error) { - keyspaceID, err := m.redirectKeyspace(constant.NullKeyspaceID, false) +func (m *GCStateManager) CompatibleLoadGCSafePoint(keyspaceID uint32) (uint64, error) { + keyspaceID, err := m.redirectKeyspace(keyspaceID, false) if err != nil { return 0, err } @@ -180,11 +180,11 @@ func (m *GCStateManager) AdvanceGCSafePoint(keyspaceID uint32, target uint64) (o // current value, it returns the current value without updating it. // This is provided for compatibility purpose, making the existing uses of the deprecated API `UpdateGCSafePoint` // still work. -func (m *GCStateManager) CompatibleUpdateGCSafePoint(target uint64) (oldGCSafePoint uint64, newGCSafePoint uint64, err error) { +func (m *GCStateManager) CompatibleUpdateGCSafePoint(keyspaceID uint32, target uint64) (oldGCSafePoint uint64, newGCSafePoint uint64, err error) { m.mu.Lock() defer m.mu.Unlock() - return m.advanceGCSafePointImpl(constant.NullKeyspaceID, target, true) + return m.advanceGCSafePointImpl(keyspaceID, target, true) } func (m *GCStateManager) advanceGCSafePointImpl(keyspaceID uint32, target uint64, compatible bool) (oldGCSafePoint uint64, newGCSafePoint uint64, err error) { @@ -305,7 +305,7 @@ func (m *GCStateManager) advanceTxnSafePointImpl(keyspaceID uint32, target uint6 } for _, barrier := range barriers { - if keyspaceID == constant.NullKeyspaceID && barrier.BarrierID == keypath.GCWorkerServiceSafePointID { + if barrier.BarrierID == keypath.GCWorkerServiceSafePointID { downgradeCompatibleMode = true continue } @@ -436,7 +436,7 @@ func (*GCStateManager) logAdvancingTxnSafePoint(keyspaceID uint32, result Advanc // GC barrier should never expire. The ttl might be rounded up, and the actual ttl is guaranteed no less than the // specified duration. // -// The barrierID must be non-empty. For NullKeyspace, "gc_worker" is a reserved name and cannot be used as a barrierID. +// The barrierID must be non-empty. "gc_worker" is a reserved name and cannot be used as a barrierID. // // The given barrierTS must be greater than or equal to the current txn safe point, or an error will be returned. // @@ -459,7 +459,7 @@ func (m *GCStateManager) SetGCBarrier(keyspaceID uint32, barrierID string, barri func (m *GCStateManager) setGCBarrierImpl(keyspaceID uint32, barrierID string, barrierTS uint64, ttl time.Duration, now time.Time) (*endpoint.GCBarrier, error) { // The barrier ID (or service ID of the service safe points) is reserved for keeping backward compatibility. - if keyspaceID == constant.NullKeyspaceID && barrierID == keypath.GCWorkerServiceSafePointID { + if barrierID == keypath.GCWorkerServiceSafePointID { return nil, errs.ErrReservedGCBarrierID.GenWithStackByArgs(barrierID) } // Disallow empty barrierID @@ -519,7 +519,7 @@ func (m *GCStateManager) DeleteGCBarrier(keyspaceID uint32, barrierID string) (* func (m *GCStateManager) deleteGCBarrierImpl(keyspaceID uint32, barrierID string) (*endpoint.GCBarrier, error) { // The barrier ID (or service ID of the service safe points) is reserved for keeping backward compatibility. - if keyspaceID == constant.NullKeyspaceID && barrierID == keypath.GCWorkerServiceSafePointID { + if barrierID == keypath.GCWorkerServiceSafePointID { return nil, errs.ErrReservedGCBarrierID.GenWithStackByArgs(barrierID) } // Disallow empty barrierID @@ -588,13 +588,11 @@ func (m *GCStateManager) getGCStateInTransaction(keyspaceID uint32, _ *endpoint. return GCState{}, err } - // For NullKeyspace, remove GC barrier whose barrierID is "gc_worker", which is only exists for providing - // compatibility with the old versions. - if keyspaceID == constant.NullKeyspaceID { - result.GCBarriers = slices.DeleteFunc(result.GCBarriers, func(b *endpoint.GCBarrier) bool { - return b.BarrierID == keypath.GCWorkerServiceSafePointID - }) - } + // Remove GC barrier whose barrierID is "gc_worker", which is only exists for providing compatibility with the old + // versions. + result.GCBarriers = slices.DeleteFunc(result.GCBarriers, func(b *endpoint.GCBarrier) bool { + return b.BarrierID == keypath.GCWorkerServiceSafePointID + }) return result, nil } @@ -700,14 +698,14 @@ func (m *GCStateManager) GetAllKeyspacesGCStates() (map[uint32]GCState, error) { // whether the `ttl` is positive or not. As the txn safe point is always less or equal to any GC barriers, we // simulate the case that the service safe point of "gc_worker" is the minimal one, and return a service safe point // with the service ID equals to "gc_worker". -// -// This function only works on the NullKeyspace. -func (m *GCStateManager) CompatibleUpdateServiceGCSafePoint(serviceID string, newServiceSafePoint uint64, ttl int64, now time.Time) (minServiceSafePoint *endpoint.ServiceSafePoint, updated bool, err error) { - keyspaceID := constant.NullKeyspaceID +func (m *GCStateManager) CompatibleUpdateServiceGCSafePoint(keyspaceID uint32, serviceID string, newServiceSafePoint uint64, ttl int64, now time.Time) (minServiceSafePoint *endpoint.ServiceSafePoint, updated bool, err error) { + keyspaceID, err = m.redirectKeyspace(keyspaceID, true) + m.mu.Lock() defer m.mu.Unlock() // TODO: After implementing the global GC barrier, redirect the invocation on "native_br" to `SetGlobalGCBarrier`. + // Note that the behavior is only for the null keyspace. if serviceID == keypath.GCWorkerServiceSafePointID { if ttl != math.MaxInt64 { return nil, false, errors.New("TTL of gc_worker's service safe point must be infinity") diff --git a/pkg/gc/gc_state_manager_test.go b/pkg/gc/gc_state_manager_test.go index b95f0a01329..853b0ffd1ff 100644 --- a/pkg/gc/gc_state_manager_test.go +++ b/pkg/gc/gc_state_manager_test.go @@ -16,6 +16,7 @@ package gc import ( "context" + "encoding/json" "math" "slices" "strconv" @@ -190,6 +191,41 @@ func (s *gcStateManagerTestSuite) deleteTiDBMinStartTS(keyspaceID uint32, instan re.NoError(err) } +func (s *gcStateManagerTestSuite) putLegacyGCWorkerServiceSafePoint(keyspaceID uint32, initialValue uint64) { + re := s.Require() + redirectedKeyspaceID, err := s.manager.redirectKeyspace(keyspaceID, false) + re.NoError(err) + re.Equal(keyspaceID, redirectedKeyspaceID, "legacy service safe point is not applicable for non-null keyspaces configured in unified GC mode") + key := keypath.ServiceGCSafePointPath(keypath.GCWorkerServiceSafePointID) + if keyspaceID != constant.NullKeyspaceID { + key = keypath.ServiceSafePointV2Path(keyspaceID, keypath.GCWorkerServiceSafePointID) + } + ssp := &endpoint.ServiceSafePoint{ + ServiceID: keypath.GCWorkerServiceSafePointID, + ExpiredAt: math.MaxInt64, + SafePoint: initialValue, + KeyspaceID: keyspaceID, + } + sspStr, err := json.Marshal(ssp) + re.NoError(err) + err = s.storage.Save(key, string(sspStr)) + re.NoError(err) +} + +func (s *gcStateManagerTestSuite) getLegacyGCWorkerServiceSafePoint(keyspaceID uint32) *endpoint.ServiceSafePoint { + re := s.Require() + // Use the reading method provided in GCStateProvider instead of reading etcd directly to avoid the potential + // mistake to read different path from that should actually be read. + _, ssps, err := s.provider.CompatibleLoadAllServiceGCSafePoints(keyspaceID) + re.NoError(err) + for _, ssp := range ssps { + if ssp.ServiceID == keypath.GCWorkerServiceSafePointID { + return ssp + } + } + return nil +} + func (s *gcStateManagerTestSuite) TestAdvanceTxnSafePointBasic() { re := s.Require() now := time.Now() @@ -337,65 +373,69 @@ func (s *gcStateManagerTestSuite) TestAdvanceGCSafePointBasic() { } } -func (s *gcStateManagerTestSuite) testGCSafePointUpdateSequentiallyImpl(loadFunc func() (uint64, error)) { +func (s *gcStateManagerTestSuite) testCompatibleGCSafePointUpdateSequentiallyImpl(keyspaceID uint32, loadFunc func(keyspaceID uint32) (uint64, error)) { re := s.Require() curGCSafePoint := uint64(0) // Update GC safe point with asc value. for id := 10; id < 20; id++ { - safePoint, err := loadFunc() + safePoint, err := loadFunc(keyspaceID) re.NoError(err) re.Equal(curGCSafePoint, safePoint) previousGCSafePoint := curGCSafePoint curGCSafePoint = uint64(id) // Blocked by txn safe point. - _, _, err = s.manager.CompatibleUpdateGCSafePoint(curGCSafePoint) + _, _, err = s.manager.CompatibleUpdateGCSafePoint(keyspaceID, curGCSafePoint) re.Error(err) re.ErrorIs(err, errs.ErrGCSafePointExceedsTxnSafePoint) - _, err = s.manager.AdvanceTxnSafePoint(constant.NullKeyspaceID, curGCSafePoint, time.Now()) + _, err = s.manager.AdvanceTxnSafePoint(keyspaceID, curGCSafePoint, time.Now()) re.NoError(err) - oldGCSafePoint, newGCSafePoint, err := s.manager.CompatibleUpdateGCSafePoint(curGCSafePoint) + oldGCSafePoint, newGCSafePoint, err := s.manager.CompatibleUpdateGCSafePoint(keyspaceID, curGCSafePoint) re.NoError(err) re.Equal(previousGCSafePoint, oldGCSafePoint) re.Equal(curGCSafePoint, newGCSafePoint) } - gcSafePoint, err := s.manager.CompatibleLoadGCSafePoint() + gcSafePoint, err := s.manager.CompatibleLoadGCSafePoint(keyspaceID) re.NoError(err) re.Equal(curGCSafePoint, gcSafePoint) // Update with smaller value should be failed. - oldGCSafePoint, newGCSafePoint, err := s.manager.CompatibleUpdateGCSafePoint(gcSafePoint - 5) + oldGCSafePoint, newGCSafePoint, err := s.manager.CompatibleUpdateGCSafePoint(keyspaceID, gcSafePoint-5) re.NoError(err) re.Equal(gcSafePoint, oldGCSafePoint) re.Equal(gcSafePoint, newGCSafePoint) - curGCSafePoint, err = s.manager.CompatibleLoadGCSafePoint() + curGCSafePoint, err = s.manager.CompatibleLoadGCSafePoint(keyspaceID) re.NoError(err) // Current GC safe point should not change since the update value was smaller re.Equal(gcSafePoint, curGCSafePoint) } func (s *gcStateManagerTestSuite) TestCompatibleUpdateGCSafePointSequentiallyWithLegacyLoad() { - s.testGCSafePointUpdateSequentiallyImpl(func() (uint64, error) { - return s.manager.CompatibleLoadGCSafePoint() - }) + for _, keyspaceID := range s.keyspacePresets.manageable { + s.testCompatibleGCSafePointUpdateSequentiallyImpl(keyspaceID, func(keyspaceID uint32) (uint64, error) { + return s.manager.CompatibleLoadGCSafePoint(keyspaceID) + }) + } } func (s *gcStateManagerTestSuite) TestCompatibleUpdateGCSafePointSequentiallyWithNewLoad() { - s.testGCSafePointUpdateSequentiallyImpl(func() (uint64, error) { - state, err := s.manager.GetGCState(constant.NullKeyspaceID) - if err != nil { - return 0, err - } - return state.GCSafePoint, nil - }) + for _, keyspaceID := range s.keyspacePresets.manageable { + s.testCompatibleGCSafePointUpdateSequentiallyImpl(keyspaceID, func(keyspaceID uint32) (uint64, error) { + state, err := s.manager.GetGCState(keyspaceID) + if err != nil { + return 0, err + } + return state.GCSafePoint, nil + }) + } } -func (s *gcStateManagerTestSuite) TestGCSafePointUpdateConcurrently() { +func (s *gcStateManagerTestSuite) testCompatibleGCSafePointUpdateConcurrentlyImpl(keyspaceID uint32) { maxGCSafePoint := uint64(1000) wg := sync.WaitGroup{} re := s.Require() // Advance txn safe point first, otherwise the GC safe point can't be advanced. - _, err := s.manager.AdvanceTxnSafePoint(constant.NullKeyspaceID, maxGCSafePoint, time.Now()) + _, err := s.manager.AdvanceTxnSafePoint(keyspaceID, maxGCSafePoint, time.Now()) re.NoError(err) ctx, cancel := context.WithCancel(context.Background()) @@ -418,12 +458,12 @@ func (s *gcStateManagerTestSuite) TestGCSafePointUpdateConcurrently() { // Mix using new and legacy API var err error if (gcSafePoint/step)%2 == 0 { - _, _, err = s.manager.AdvanceGCSafePoint(constant.NullKeyspaceID, gcSafePoint) + _, _, err = s.manager.AdvanceGCSafePoint(keyspaceID, gcSafePoint) if err != nil && errors.ErrorEqual(err, errs.ErrDecreasingGCSafePoint) { err = nil } } else { - _, _, err = s.manager.CompatibleUpdateGCSafePoint(gcSafePoint) + _, _, err = s.manager.CompatibleUpdateGCSafePoint(keyspaceID, gcSafePoint) } if err != nil { errCh <- err @@ -440,12 +480,19 @@ func (s *gcStateManagerTestSuite) TestGCSafePointUpdateConcurrently() { re.NoError(err) default: } - gcSafePoint, err := s.manager.CompatibleLoadGCSafePoint() + gcSafePoint, err := s.manager.CompatibleLoadGCSafePoint(keyspaceID) re.NoError(err) re.Equal(maxGCSafePoint, gcSafePoint) } -func (s *gcStateManagerTestSuite) TestLegacyServiceGCSafePointUpdate() { +func (s *gcStateManagerTestSuite) TestCompatibleGCSafePointUpdateConcurrently() { + for _, keyspaceID := range s.keyspacePresets.manageable { + s.testCompatibleGCSafePointUpdateConcurrentlyImpl(keyspaceID) + } +} + +func (s *gcStateManagerTestSuite) TestCompatibleServiceGCSafePointUpdateNullKeyspace() { + keyspaceID := constant.NullKeyspaceID re := s.Require() gcWorkerServiceID := "gc_worker" cdcServiceID := "cdc" @@ -459,7 +506,7 @@ func (s *gcStateManagerTestSuite) TestLegacyServiceGCSafePointUpdate() { // Updating the service safe point for cdc to 10 should success go func() { defer wg.Done() - min, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(cdcServiceID, cdcServiceSafePoint, 10000, time.Now()) + min, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, cdcServiceID, cdcServiceSafePoint, 10000, time.Now()) re.NoError(err) re.True(updated) // The service will init the service safepoint to 0(<10 for cdc) for gc_worker. @@ -469,7 +516,7 @@ func (s *gcStateManagerTestSuite) TestLegacyServiceGCSafePointUpdate() { // Updating the service safe point for br to 15 should success go func() { defer wg.Done() - min, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(brServiceID, brSafePoint, 10000, time.Now()) + min, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, brServiceID, brSafePoint, 10000, time.Now()) re.NoError(err) re.True(updated) // the service will init the service safepoint to 0(<10 for cdc) for gc_worker. @@ -480,7 +527,7 @@ func (s *gcStateManagerTestSuite) TestLegacyServiceGCSafePointUpdate() { go func() { defer wg.Done() // update with valid ttl for gc_worker should be success. - min, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) + min, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) re.NoError(err) re.True(updated) // the current min safepoint should be 8 for gc_worker(cdc 10) @@ -491,7 +538,7 @@ func (s *gcStateManagerTestSuite) TestLegacyServiceGCSafePointUpdate() { go func() { defer wg.Done() // Updating the service safe point of gc_worker's service with ttl not infinity should be failed. - _, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(gcWorkerServiceID, 10000, 10, time.Now()) + _, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, gcWorkerServiceID, 10000, 10, time.Now()) re.Error(err) re.False(updated) }() @@ -500,7 +547,7 @@ func (s *gcStateManagerTestSuite) TestLegacyServiceGCSafePointUpdate() { go func() { defer wg.Done() brTTL := int64(-100) - _, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(brServiceID, uint64(10000), brTTL, time.Now()) + _, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, brServiceID, uint64(10000), brTTL, time.Now()) re.NoError(err) re.False(updated) }() @@ -508,7 +555,7 @@ func (s *gcStateManagerTestSuite) TestLegacyServiceGCSafePointUpdate() { wg.Wait() // Updating the service safe point to 15(>10 for cdc) for gc_worker gcWorkerSafePoint = uint64(15) - min, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) + min, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) re.NoError(err) re.True(updated) re.Equal(cdcServiceID, min.ServiceID) @@ -517,44 +564,46 @@ func (s *gcStateManagerTestSuite) TestLegacyServiceGCSafePointUpdate() { // The value shouldn't be updated with current service safe point smaller than the min safe point. brTTL := int64(100) brSafePoint = min.SafePoint - 5 - min, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(brServiceID, brSafePoint, brTTL, time.Now()) + min, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, brServiceID, brSafePoint, brTTL, time.Now()) re.NoError(err) re.False(updated) brSafePoint = min.SafePoint + 10 - _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(brServiceID, brSafePoint, brTTL, time.Now()) + _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, brServiceID, brSafePoint, brTTL, time.Now()) re.NoError(err) re.True(updated) } -func (s *gcStateManagerTestSuite) TestLegacyServiceGCSafePointRoundingTTL() { +func (s *gcStateManagerTestSuite) TestCompatibleServiceGCSafePointRoundingTTL() { re := s.Require() var maxTTL int64 = 9223372036 - _, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint("svc1", 10, maxTTL, time.Now()) - re.NoError(err) - re.True(updated) + for _, keyspaceID := range s.keyspacePresets.manageable { + _, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc1", 10, maxTTL, time.Now()) + re.NoError(err) + re.True(updated) - state, err := s.manager.GetGCState(constant.NullKeyspaceID) - re.NoError(err) - re.Len(state.GCBarriers, 1) - re.Equal("svc1", state.GCBarriers[0].BarrierID) - re.NotNil(state.GCBarriers[0].ExpirationTime) - // The given `maxTTL` is valid but super large. - re.True(state.GCBarriers[0].ExpirationTime.After(time.Now().Add(time.Hour*24*365*10)), state.GCBarriers[0].ExpirationTime) + state, err := s.manager.GetGCState(keyspaceID) + re.NoError(err) + re.Len(state.GCBarriers, 1) + re.Equal("svc1", state.GCBarriers[0].BarrierID) + re.NotNil(state.GCBarriers[0].ExpirationTime) + // The given `maxTTL` is valid but super large. + re.True(state.GCBarriers[0].ExpirationTime.After(time.Now().Add(time.Hour*24*365*10)), state.GCBarriers[0].ExpirationTime) - _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("svc1", 10, maxTTL+1, time.Now()) - re.NoError(err) - re.True(updated) + _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc1", 10, maxTTL+1, time.Now()) + re.NoError(err) + re.True(updated) - state, err = s.manager.GetGCState(constant.NullKeyspaceID) - re.NoError(err) - re.Len(state.GCBarriers, 1) - re.Equal("svc1", state.GCBarriers[0].BarrierID) - re.Nil(state.GCBarriers[0].ExpirationTime) - // Nil in GCBarrier.ExpirationTime represents never expires. - re.False(state.GCBarriers[0].IsExpired(time.Now().Add(time.Hour*24*365*10)), state.GCBarriers[0]) + state, err = s.manager.GetGCState(keyspaceID) + re.NoError(err) + re.Len(state.GCBarriers, 1) + re.Equal("svc1", state.GCBarriers[0].BarrierID) + re.Nil(state.GCBarriers[0].ExpirationTime) + // Nil in GCBarrier.ExpirationTime represents never expires. + re.False(state.GCBarriers[0].IsExpired(time.Now().Add(time.Hour*24*365*10)), state.GCBarriers[0]) + } } func (s *gcStateManagerTestSuite) getGCBarrier(keyspaceID uint32, barrierID string) *endpoint.GCBarrier { @@ -836,19 +885,21 @@ func (s *gcStateManagerTestSuite) TestGCBarriers() { re.ErrorIs(err, errs.ErrKeyspaceNotFound) } - // Rejects reserved barrier ID: rejects "gc_worker" in NullKeyspace. - _, err := s.manager.SetGCBarrier(constant.NullKeyspaceID, "gc_worker", 100, time.Hour, now) - re.Error(err) - re.ErrorIs(err, errs.ErrReservedGCBarrierID) - re.Nil(s.getGCBarrier(constant.NullKeyspaceID, "gc_worker")) - _, err = s.manager.DeleteGCBarrier(constant.NullKeyspaceID, "gc_worker") - re.Error(err) - re.ErrorIs(err, errs.ErrReservedGCBarrierID) + // Rejects reserved barrier ID: rejects "gc_worker". + for _, keyspaceID := range s.keyspacePresets.all { + _, err := s.manager.SetGCBarrier(keyspaceID, "gc_worker", 100, time.Hour, now) + re.Error(err) + re.ErrorIs(err, errs.ErrReservedGCBarrierID) + re.Nil(s.getGCBarrier(keyspaceID, "gc_worker")) + _, err = s.manager.DeleteGCBarrier(keyspaceID, "gc_worker") + re.Error(err) + re.ErrorIs(err, errs.ErrReservedGCBarrierID) + } // Isolated between different keyspaces. ks1 := s.keyspacePresets.manageable[0] ks2 := s.keyspacePresets.manageable[1] - _, err = s.manager.SetGCBarrier(ks1, "b1", 200, time.Hour, now) + _, err := s.manager.SetGCBarrier(ks1, "b1", 200, time.Hour, now) re.NoError(err) expected := endpoint.NewGCBarrier("b1", 200, ptime(now.Add(time.Hour))) re.Equal(expected, s.getGCBarrier(ks1, "b1")) @@ -955,30 +1006,30 @@ func (s *gcStateManagerTestSuite) TestTiDBMinStartTS() { } } -func (s *gcStateManagerTestSuite) TestServiceGCSafePointCompatibility() { +func (s *gcStateManagerTestSuite) testServiceGCSafePointCompatibilityImpl(keyspaceID uint32) { re := s.Require() var nowUnix int64 = 1741584577 now := time.Unix(nowUnix, 0) // Service safe points & GC barriers shares the same data storage and are mutually convertable. - minSsp, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint("svc1", 10, math.MaxInt64, now) + minSsp, updated, err := s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc1", 10, math.MaxInt64, now) re.NoError(err) re.True(updated) re.Equal(uint64(0), minSsp.SafePoint) re.Equal("gc_worker", minSsp.ServiceID) - res, err := s.manager.AdvanceTxnSafePoint(constant.NullKeyspaceID, 15, now) + res, err := s.manager.AdvanceTxnSafePoint(keyspaceID, 15, now) re.NoError(err) re.Equal(uint64(10), res.NewTxnSafePoint) expected := endpoint.NewGCBarrier("svc1", 10, nil) - re.Equal(expected, s.getGCBarrier(constant.NullKeyspaceID, "svc1")) + re.Equal(expected, s.getGCBarrier(keyspaceID, "svc1")) // SetGCBarrier can also affect service safe points. - _, err = s.manager.SetGCBarrier(constant.NullKeyspaceID, "svc1", 15, time.Hour, now) + _, err = s.manager.SetGCBarrier(keyspaceID, "svc1", 15, time.Hour, now) re.NoError(err) - _, allSsp, err := s.provider.CompatibleLoadAllServiceGCSafePoints() + _, allSsp, err := s.provider.CompatibleLoadAllServiceGCSafePoints(keyspaceID) re.NoError(err) re.Len(allSsp, 1) re.Equal("svc1", allSsp[0].ServiceID) @@ -986,84 +1037,84 @@ func (s *gcStateManagerTestSuite) TestServiceGCSafePointCompatibility() { re.Equal(nowUnix+3600, allSsp[0].ExpiredAt) // Disallow decreasing behind the txn safe point. But it doesn't return error. - minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("svc1", 8, math.MaxInt64, now) + minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc1", 8, math.MaxInt64, now) re.NoError(err) re.False(updated) re.Equal(uint64(10), minSsp.SafePoint) expected = endpoint.NewGCBarrier("svc1", 15, ptime(now.Add(time.Hour))) - re.Equal(expected, s.getGCBarrier(constant.NullKeyspaceID, "svc1")) + re.Equal(expected, s.getGCBarrier(keyspaceID, "svc1")) // Disallow inserting new service safe point before the txn safe point. - minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("svc2", 8, math.MaxInt64, now) + minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc2", 8, math.MaxInt64, now) re.NoError(err) re.False(updated) re.Equal(uint64(10), minSsp.SafePoint) - re.Nil(s.getGCBarrier(constant.NullKeyspaceID, "svc2")) + re.Nil(s.getGCBarrier(keyspaceID, "svc2")) // But decreasing a service safe point to a value larger than the current txn safe point is allowed. Note that this // behavior is not completely the same as old versions. - minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("svc1", 12, math.MaxInt64, now) + minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc1", 12, math.MaxInt64, now) re.NoError(err) re.True(updated) re.Equal(uint64(10), minSsp.SafePoint) expected = endpoint.NewGCBarrier("svc1", 12, nil) - re.Equal(expected, s.getGCBarrier(constant.NullKeyspaceID, "svc1")) + re.Equal(expected, s.getGCBarrier(keyspaceID, "svc1")) // Allows setting different TTL. - _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("svc1", 12, 3600, now) + _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc1", 12, 3600, now) re.NoError(err) re.True(updated) - _, allSsp, err = s.provider.CompatibleLoadAllServiceGCSafePoints() + _, allSsp, err = s.provider.CompatibleLoadAllServiceGCSafePoints(keyspaceID) re.NoError(err) re.Len(allSsp, 1) re.Equal("svc1", allSsp[0].ServiceID) re.Equal(uint64(12), allSsp[0].SafePoint) re.Equal(nowUnix+3600, allSsp[0].ExpiredAt) - _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("svc1", 12, 3600, now.Add(time.Hour)) + _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc1", 12, 3600, now.Add(time.Hour)) re.NoError(err) re.True(updated) - _, allSsp, err = s.provider.CompatibleLoadAllServiceGCSafePoints() + _, allSsp, err = s.provider.CompatibleLoadAllServiceGCSafePoints(keyspaceID) re.NoError(err) re.Len(allSsp, 1) re.Equal(nowUnix+7200, allSsp[0].ExpiredAt) // Internally calls AdvanceTxnSafePoint when simulating updating "gc_worker". - _, _, err = s.manager.CompatibleUpdateServiceGCSafePoint("gc_worker", 20, 3600, now) + _, _, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "gc_worker", 20, 3600, now) // Cannot use finite TTL for "gc_worker". re.Error(err) - minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("gc_worker", 20, math.MaxInt64, now) + minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "gc_worker", 20, math.MaxInt64, now) re.NoError(err) re.True(updated) re.Equal(uint64(12), minSsp.SafePoint) re.Equal("svc1", minSsp.ServiceID) - s.checkTxnSafePoint(constant.NullKeyspaceID, 12) + s.checkTxnSafePoint(keyspaceID, 12) // Deleting service safe point by passing zero TTL - _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("svc1", 12, 0, now) + _, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc1", 12, 0, now) re.NoError(err) // Deleting is regarded as not-updating. This is consistent with the behavior of the old UpdateServiceGCSafePoint API. re.False(updated) // And add a TiDBMinStartTS. Then it should also block updating "gc_worker". // This behavior doesn't exist in old UpdateServiceGCSafePoint API, and is new here. In this case, it returns a // simulated service safe point which doesn't actually exist. - s.setTiDBMinStartTS(constant.NullKeyspaceID, "instance1", 14) - minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("gc_worker", 20, math.MaxInt64, now) + s.setTiDBMinStartTS(keyspaceID, "instance1", 14) + minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "gc_worker", 20, math.MaxInt64, now) re.NoError(err) re.True(updated) re.Equal(uint64(14), minSsp.SafePoint) re.Equal("tidb_min_start_ts_instance1", minSsp.ServiceID) - s.checkTxnSafePoint(constant.NullKeyspaceID, 14) + s.checkTxnSafePoint(keyspaceID, 14) // Delete the TiDBMinStartTS. - s.deleteTiDBMinStartTS(constant.NullKeyspaceID, "instance1") + s.deleteTiDBMinStartTS(keyspaceID, "instance1") // Then updating "gc_worker" won't be blocked. - minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("gc_worker", 20, math.MaxInt64, now) + minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "gc_worker", 20, math.MaxInt64, now) re.NoError(err) re.True(updated) re.Equal(uint64(20), minSsp.SafePoint) re.Equal("gc_worker", minSsp.ServiceID) - s.checkTxnSafePoint(constant.NullKeyspaceID, 20) + s.checkTxnSafePoint(keyspaceID, 20) // If there's already old data written by the old version, there will exist a persisted "gc_worker" service safe // point, in which case AdvanceTxnSafePoint needs to update it as well for guaranteeing the safety of @@ -1071,20 +1122,20 @@ func (s *gcStateManagerTestSuite) TestServiceGCSafePointCompatibility() { // of "gc_worker" is updated to the same value as the txn safe point. // This behavior only exist in the NullKeyspace. re.NoError(s.provider.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error { - return wb.SetGCBarrier(constant.NullKeyspaceID, endpoint.NewGCBarrier("gc_worker", 20, nil)) + return wb.SetGCBarrier(keyspaceID, endpoint.NewGCBarrier("gc_worker", 20, nil)) })) - minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("svc1", 25, math.MaxInt64, now) + minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "svc1", 25, math.MaxInt64, now) re.NoError(err) re.True(updated) re.Equal(uint64(20), minSsp.SafePoint) re.Equal("gc_worker", minSsp.ServiceID) - minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint("gc_worker", 28, math.MaxInt64, now) + minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(keyspaceID, "gc_worker", 28, math.MaxInt64, now) re.NoError(err) re.True(updated) re.Equal(uint64(25), minSsp.SafePoint) re.NotEqual("gc_worker", minSsp.ServiceID) - _, allSsp, err = s.provider.CompatibleLoadAllServiceGCSafePoints() + _, allSsp, err = s.provider.CompatibleLoadAllServiceGCSafePoints(keyspaceID) re.NoError(err) re.Len(allSsp, 2) re.Equal("gc_worker", allSsp[0].ServiceID) @@ -1092,42 +1143,51 @@ func (s *gcStateManagerTestSuite) TestServiceGCSafePointCompatibility() { re.Equal("svc1", allSsp[1].ServiceID) re.Equal(uint64(25), allSsp[1].SafePoint) - res, err = s.manager.AdvanceTxnSafePoint(constant.NullKeyspaceID, 29, now) + res, err = s.manager.AdvanceTxnSafePoint(keyspaceID, 29, now) re.NoError(err) re.Equal(uint64(25), res.NewTxnSafePoint) re.Contains(res.BlockerDescription, `BarrierID: "svc1"`) - _, allSsp, err = s.provider.CompatibleLoadAllServiceGCSafePoints() + _, allSsp, err = s.provider.CompatibleLoadAllServiceGCSafePoints(keyspaceID) re.NoError(err) re.Len(allSsp, 2) re.Equal("gc_worker", allSsp[0].ServiceID) re.Equal(uint64(25), allSsp[0].SafePoint) // The service safe point "gc_worker" is ignored by GetGCState. - allBarriers := s.getAllGCBarriers(constant.NullKeyspaceID) + allBarriers := s.getAllGCBarriers(keyspaceID) re.Len(allBarriers, 1) re.Equal("svc1", allBarriers[0].BarrierID) // The service safe point can't be controlled by SetGCBarrier and DeleteGCBarrier. - _, err = s.manager.SetGCBarrier(constant.NullKeyspaceID, "gc_worker", 30, time.Duration(math.MaxInt64), now) + _, err = s.manager.SetGCBarrier(keyspaceID, "gc_worker", 30, time.Duration(math.MaxInt64), now) re.Error(err) re.ErrorIs(err, errs.ErrReservedGCBarrierID) - _, err = s.manager.DeleteGCBarrier(constant.NullKeyspaceID, "gc_worker") + _, err = s.manager.DeleteGCBarrier(keyspaceID, "gc_worker") re.Error(err) re.ErrorIs(err, errs.ErrReservedGCBarrierID) - // The same behavior doesn't exist in other keyspaces with keyspace-level GC enabled. - _, err = s.manager.SetGCBarrier(2, "gc_worker", 10, time.Hour, now) - re.NoError(err) - res, err = s.manager.AdvanceTxnSafePoint(2, 15, now) - re.NoError(err) - re.Equal(uint64(10), res.NewTxnSafePoint) - re.Contains(res.BlockerDescription, `BarrierID: "gc_worker"`) - _, err = s.manager.DeleteGCBarrier(2, "gc_worker") - re.NoError(err) - res, err = s.manager.AdvanceTxnSafePoint(2, 15, now) - re.NoError(err) - re.Equal(uint64(15), res.NewTxnSafePoint) - re.Empty(res.BlockerDescription) + // It does not affect other self-manageable keyspaces. + for _, anotherKeyspaceID := range s.keyspacePresets.manageable { + if anotherKeyspaceID == keyspaceID { + continue + } + re.Equal(uint64(25), s.getGCBarrier(keyspaceID, "svc1").BarrierTS) + res, err = s.manager.AdvanceTxnSafePoint(anotherKeyspaceID, 30, now) + re.NoError(err) + re.Equal(uint64(30), res.NewTxnSafePoint) + minSsp, updated, err = s.manager.CompatibleUpdateServiceGCSafePoint(anotherKeyspaceID, "gc_worker", 35, math.MaxInt64, now) + re.NoError(err) + re.True(updated) + re.Equal(uint64(35), minSsp.SafePoint) + } +} + +func (s *gcStateManagerTestSuite) TestServiceGCSafePointCompatibilityForNullKeyspace() { + s.testServiceGCSafePointCompatibilityImpl(constant.NullKeyspaceID) +} + +func (s *gcStateManagerTestSuite) TestServiceGCSafePointCompatibilityForNonNullKeyspace() { + s.testServiceGCSafePointCompatibilityImpl(2) } func (s *gcStateManagerTestSuite) TestRedirectKeyspace() { @@ -1356,3 +1416,59 @@ func (s *gcStateManagerTestSuite) TestWeakenedConstraints() { s.checkTxnSafePoint(keyspaceID, 30) } } + +func (s *gcStateManagerTestSuite) testDowngradeCompatibility(keyspaceID uint32) { + re := s.Require() + now := time.Now() + + // When downgrade compatible mode of AdvanceTxnSafePoint is triggerred, the "gc_worker"'s service safe point + // will be updated synchronized with the txn safe point. + s.putLegacyGCWorkerServiceSafePoint(keyspaceID, 0) + for _, target := range []uint64{10, 20} { + res, err := s.manager.AdvanceTxnSafePoint(keyspaceID, target, now) + re.NoError(err) + re.Equal(target-10, res.OldTxnSafePoint) + re.Equal(target, res.NewTxnSafePoint) + re.Empty(res.BlockerDescription) + re.Equal(target, s.getLegacyGCWorkerServiceSafePoint(keyspaceID).SafePoint) + } + + // Allow decreasing. + s.putLegacyGCWorkerServiceSafePoint(keyspaceID, 30) + res, err := s.manager.AdvanceTxnSafePoint(keyspaceID, 25, now) + re.NoError(err) + re.Equal(uint64(20), res.OldTxnSafePoint) + re.Equal(uint64(25), res.NewTxnSafePoint) + re.Empty(res.BlockerDescription) + re.Equal(uint64(25), s.getLegacyGCWorkerServiceSafePoint(keyspaceID).SafePoint) + + // Not visible by GetGCStates or GetAllKeyspacesGCStates. + gcState, err := s.manager.GetGCState(keyspaceID) + re.NoError(err) + re.Empty(gcState.GCBarriers) + allGCStates, err := s.manager.GetAllKeyspacesGCStates() + re.NoError(err) + re.Empty(allGCStates[keyspaceID].GCBarriers) + + // And it works correctly when there are other valid GC barriers. + _, err = s.manager.SetGCBarrier(keyspaceID, "b1", 40, time.Hour, now) + re.NoError(err) + gcState, err = s.manager.GetGCState(keyspaceID) + re.NoError(err) + re.Len(gcState.GCBarriers, 1) + re.Equal("b1", gcState.GCBarriers[0].BarrierID) + re.Equal(uint64(40), gcState.GCBarriers[0].BarrierTS) + allGCStates, err = s.manager.GetAllKeyspacesGCStates() + re.NoError(err) + re.Len(allGCStates[keyspaceID].GCBarriers, 1) + re.Equal("b1", allGCStates[keyspaceID].GCBarriers[0].BarrierID) + re.Equal(uint64(40), allGCStates[keyspaceID].GCBarriers[0].BarrierTS) +} + +func (s *gcStateManagerTestSuite) TestDowngradeCompatibilityForNullKeyspace() { + s.testDowngradeCompatibility(constant.NullKeyspaceID) +} + +func (s *gcStateManagerTestSuite) TestDowngradeCompatibilityForNonNullKeyspace() { + s.testDowngradeCompatibility(2) +} diff --git a/pkg/gc/safepoint.go b/pkg/gc/safepoint.go deleted file mode 100644 index 5bc64ae9a90..00000000000 --- a/pkg/gc/safepoint.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gc - -import ( - "math" - "time" - - "github.com/pingcap/errors" - - "github.com/tikv/pd/pkg/storage/endpoint" - "github.com/tikv/pd/pkg/utils/syncutil" - "github.com/tikv/pd/server/config" -) - -var blockGCSafePointErrmsg = "don't allow update gc safe point v1." -var blockServiceSafepointErrmsg = "don't allow update service safe point v1." - -// SafePointManager is the manager for safePoint of GC and services. -type SafePointManager struct { - gcLock syncutil.Mutex - serviceGCLock syncutil.Mutex - store endpoint.GCSafePointStorage - cfg config.PDServerConfig -} - -// NewSafePointManager creates a SafePointManager of GC and services. -func NewSafePointManager(store endpoint.GCSafePointStorage, cfg config.PDServerConfig) *SafePointManager { - return &SafePointManager{store: store, cfg: cfg} -} - -// LoadGCSafePoint loads current GC safe point from storage. -func (manager *SafePointManager) LoadGCSafePoint() (uint64, error) { - return manager.store.LoadGCSafePoint() -} - -// UpdateGCSafePoint updates the safepoint if it is greater than the previous one -// it returns the old safepoint in the storage. -func (manager *SafePointManager) UpdateGCSafePoint(newSafePoint uint64) (oldSafePoint uint64, err error) { - manager.gcLock.Lock() - defer manager.gcLock.Unlock() - // TODO: cache the safepoint in the storage. - oldSafePoint, err = manager.store.LoadGCSafePoint() - if err != nil { - return - } - if manager.cfg.BlockSafePointV1 { - err = errors.New(blockGCSafePointErrmsg) - return - } - - if oldSafePoint >= newSafePoint { - return - } - err = manager.store.SaveGCSafePoint(newSafePoint) - if err == nil { - gcSafePointGauge.WithLabelValues("gc_safepoint").Set(float64(newSafePoint)) - } - return -} - -// UpdateServiceGCSafePoint update the safepoint for a specific service. -func (manager *SafePointManager) UpdateServiceGCSafePoint(serviceID string, newSafePoint uint64, ttl int64, now time.Time) (minServiceSafePoint *endpoint.ServiceSafePoint, updated bool, err error) { - if manager.cfg.BlockSafePointV1 { - return nil, false, errors.New(blockServiceSafepointErrmsg) - } - manager.serviceGCLock.Lock() - defer manager.serviceGCLock.Unlock() - minServiceSafePoint, err = manager.store.LoadMinServiceGCSafePoint(now) - if err != nil || ttl <= 0 || newSafePoint < minServiceSafePoint.SafePoint { - return minServiceSafePoint, false, err - } - - ssp := &endpoint.ServiceSafePoint{ - ServiceID: serviceID, - ExpiredAt: now.Unix() + ttl, - SafePoint: newSafePoint, - } - if math.MaxInt64-now.Unix() <= ttl { - ssp.ExpiredAt = math.MaxInt64 - } - if err := manager.store.SaveServiceGCSafePoint(ssp); err != nil { - return nil, false, err - } - - // If the min safePoint is updated, load the next one. - if serviceID == minServiceSafePoint.ServiceID { - minServiceSafePoint, err = manager.store.LoadMinServiceGCSafePoint(now) - } - return minServiceSafePoint, true, err -} diff --git a/pkg/gc/safepoint_test.go b/pkg/gc/safepoint_test.go deleted file mode 100644 index 21e79fab343..00000000000 --- a/pkg/gc/safepoint_test.go +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gc - -import ( - "math" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/tikv/pd/pkg/storage/endpoint" - "github.com/tikv/pd/pkg/storage/kv" - "github.com/tikv/pd/server/config" -) - -func newGCStorage() endpoint.GCSafePointStorage { - return endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) -} - -func TestGCSafePointUpdateSequentially(t *testing.T) { - gcSafePointManager := NewSafePointManager(newGCStorage(), config.PDServerConfig{}) - re := require.New(t) - curSafePoint := uint64(0) - // update gc safePoint with asc value. - for id := 10; id < 20; id++ { - safePoint, err := gcSafePointManager.LoadGCSafePoint() - re.NoError(err) - re.Equal(curSafePoint, safePoint) - previousSafePoint := curSafePoint - curSafePoint = uint64(id) - oldSafePoint, err := gcSafePointManager.UpdateGCSafePoint(curSafePoint) - re.NoError(err) - re.Equal(previousSafePoint, oldSafePoint) - } - - safePoint, err := gcSafePointManager.LoadGCSafePoint() - re.NoError(err) - re.Equal(curSafePoint, safePoint) - // update with smaller value should be failed. - oldSafePoint, err := gcSafePointManager.UpdateGCSafePoint(safePoint - 5) - re.NoError(err) - re.Equal(safePoint, oldSafePoint) - curSafePoint, err = gcSafePointManager.LoadGCSafePoint() - re.NoError(err) - // current safePoint should not change since the update value was smaller - re.Equal(safePoint, curSafePoint) -} - -func TestGCSafePointUpdateCurrently(t *testing.T) { - gcSafePointManager := NewSafePointManager(newGCStorage(), config.PDServerConfig{}) - maxSafePoint := uint64(1000) - wg := sync.WaitGroup{} - re := require.New(t) - - // update gc safePoint concurrently - for id := range 20 { - wg.Add(1) - go func(step uint64) { - for safePoint := step; safePoint <= maxSafePoint; safePoint += step { - _, err := gcSafePointManager.UpdateGCSafePoint(safePoint) - re.NoError(err) - } - wg.Done() - }(uint64(id + 1)) - } - wg.Wait() - safePoint, err := gcSafePointManager.LoadGCSafePoint() - re.NoError(err) - re.Equal(maxSafePoint, safePoint) -} - -func TestServiceGCSafePointUpdate(t *testing.T) { - re := require.New(t) - manager := NewSafePointManager(newGCStorage(), config.PDServerConfig{}) - gcWorkerServiceID := "gc_worker" - cdcServiceID := "cdc" - brServiceID := "br" - cdcServiceSafePoint := uint64(10) - gcWorkerSafePoint := uint64(8) - brSafePoint := uint64(15) - - wg := sync.WaitGroup{} - wg.Add(5) - // update the safepoint for cdc to 10 should success - go func() { - defer wg.Done() - min, updated, err := manager.UpdateServiceGCSafePoint(cdcServiceID, cdcServiceSafePoint, 10000, time.Now()) - re.NoError(err) - re.True(updated) - // the service will init the service safepoint to 0(<10 for cdc) for gc_worker. - re.Equal(gcWorkerServiceID, min.ServiceID) - }() - - // update the safepoint for br to 15 should success - go func() { - defer wg.Done() - min, updated, err := manager.UpdateServiceGCSafePoint(brServiceID, brSafePoint, 10000, time.Now()) - re.NoError(err) - re.True(updated) - // the service will init the service safepoint to 0(<10 for cdc) for gc_worker. - re.Equal(gcWorkerServiceID, min.ServiceID) - }() - - // update safepoint to 8 for gc_worker should be success - go func() { - defer wg.Done() - // update with valid ttl for gc_worker should be success. - min, updated, err := manager.UpdateServiceGCSafePoint(gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) - re.NoError(err) - re.True(updated) - // the current min safepoint should be 8 for gc_worker(cdc 10) - re.Equal(gcWorkerSafePoint, min.SafePoint) - re.Equal(gcWorkerServiceID, min.ServiceID) - }() - - go func() { - defer wg.Done() - // update safepoint of gc_worker's service with ttl not infinity should be failed. - _, updated, err := manager.UpdateServiceGCSafePoint(gcWorkerServiceID, 10000, 10, time.Now()) - re.Error(err) - re.False(updated) - }() - - // update safepoint with negative ttl should be failed. - go func() { - defer wg.Done() - brTTL := int64(-100) - _, updated, err := manager.UpdateServiceGCSafePoint(brServiceID, uint64(10000), brTTL, time.Now()) - re.NoError(err) - re.False(updated) - }() - - wg.Wait() - // update safepoint to 15(>10 for cdc) for gc_worker - gcWorkerSafePoint = uint64(15) - min, updated, err := manager.UpdateServiceGCSafePoint(gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) - re.NoError(err) - re.True(updated) - re.Equal(cdcServiceID, min.ServiceID) - re.Equal(cdcServiceSafePoint, min.SafePoint) - - // the value shouldn't be updated with current safepoint smaller than the min safepoint. - brTTL := int64(100) - brSafePoint = min.SafePoint - 5 - min, updated, err = manager.UpdateServiceGCSafePoint(brServiceID, brSafePoint, brTTL, time.Now()) - re.NoError(err) - re.False(updated) - - brSafePoint = min.SafePoint + 10 - _, updated, err = manager.UpdateServiceGCSafePoint(brServiceID, brSafePoint, brTTL, time.Now()) - re.NoError(err) - re.True(updated) -} - -func TestBlockUpdateSafePointV1(t *testing.T) { - re := require.New(t) - manager := NewSafePointManager(newGCStorage(), config.PDServerConfig{BlockSafePointV1: true}) - gcworkerServiceID := "gc_worker" - gcWorkerSafePoint := uint64(8) - - min, updated, err := manager.UpdateServiceGCSafePoint(gcworkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) - re.Error(err, blockServiceSafepointErrmsg) - re.Equal(err.Error(), blockServiceSafepointErrmsg) - re.False(updated) - re.Nil(min) - - oldSafePoint, err := manager.UpdateGCSafePoint(gcWorkerSafePoint) - re.Error(err) - re.Equal(err.Error(), blockGCSafePointErrmsg) - - re.Equal(uint64(0), oldSafePoint) -} diff --git a/pkg/gc/safepoint_v2.go b/pkg/gc/safepoint_v2.go deleted file mode 100644 index 3ecae974785..00000000000 --- a/pkg/gc/safepoint_v2.go +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2023 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gc - -import ( - "context" - "time" - - "go.uber.org/zap" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/kvproto/pkg/keyspacepb" - "github.com/pingcap/log" - - "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/keyspace" - "github.com/tikv/pd/pkg/slice" - "github.com/tikv/pd/pkg/storage/endpoint" - "github.com/tikv/pd/pkg/storage/kv" - "github.com/tikv/pd/pkg/utils/syncutil" -) - -var ( - // allowUpdateSafePoint specifies under which states is a keyspace allowed to update it's gc & service safe points. - allowUpdateSafePoint = []keyspacepb.KeyspaceState{ - keyspacepb.KeyspaceState_ENABLED, - keyspacepb.KeyspaceState_DISABLED, - } -) - -// SafePointV2Manager is the manager for GCSafePointV2 and ServiceSafePointV2. -type SafePointV2Manager struct { - *syncutil.LockGroup - ctx context.Context - // keyspaceStorage stores keyspace meta. - keyspaceStorage endpoint.KeyspaceStorage - // v2Storage is the storage GCSafePointV2 and ServiceSafePointV2. - v2Storage endpoint.SafePointV2Storage - // v1Storage is the storage for v1 format GCSafePoint and ServiceGCSafePoint, it's used during pd update. - v1Storage endpoint.GCSafePointStorage -} - -// NewSafePointManagerV2 returns a new SafePointV2Manager. -func NewSafePointManagerV2( - ctx context.Context, - keyspaceStore endpoint.KeyspaceStorage, - v2Storage endpoint.SafePointV2Storage, - v1Storage endpoint.GCSafePointStorage, -) *SafePointV2Manager { - return &SafePointV2Manager{ - ctx: ctx, - LockGroup: syncutil.NewLockGroup(syncutil.WithHash(keyspace.MaskKeyspaceID)), - keyspaceStorage: keyspaceStore, - v2Storage: v2Storage, - v1Storage: v1Storage, - } -} - -// LoadGCSafePoint returns GCSafePointV2 of keyspaceID. -func (manager *SafePointV2Manager) LoadGCSafePoint(keyspaceID uint32) (*endpoint.GCSafePointV2, error) { - manager.Lock(keyspaceID) - defer manager.Unlock(keyspaceID) - // Check if keyspace is valid to load. - if err := manager.checkKeyspace(keyspaceID, false); err != nil { - return nil, err - } - gcSafePoint, err := manager.getGCSafePoint(keyspaceID) - if err != nil { - log.Warn("failed to load gc safe point", - zap.Uint32("keyspace-id", keyspaceID), - zap.Error(err), - ) - return nil, err - } - return gcSafePoint, nil -} - -// checkKeyspace check if target keyspace exists, and if request is a update request, -// also check if keyspace state allows for update. -func (manager *SafePointV2Manager) checkKeyspace(keyspaceID uint32, updateRequest bool) error { - failpoint.Inject("checkKeyspace", func() { - failpoint.Return(nil) - }) - - err := manager.keyspaceStorage.RunInTxn(manager.ctx, func(txn kv.Txn) error { - meta, err := manager.keyspaceStorage.LoadKeyspaceMeta(txn, keyspaceID) - if err != nil { - return err - } - // If a keyspace does not exist, then loading its gc safe point is prohibited. - if meta == nil { - return errs.ErrKeyspaceNotFound - } - // If keyspace's state does not permit updating safe point, we return error. - if updateRequest && !slice.Contains(allowUpdateSafePoint, meta.GetState()) { - return errors.Errorf("cannot update keyspace that's %s", meta.GetState().String()) - } - return nil - }) - if err != nil { - log.Warn("check keyspace failed", - zap.Uint32("keyspace-id", keyspaceID), - zap.Error(err), - ) - } - return err -} - -// getGCSafePoint first try to load gc safepoint from v2 storage, if failed, load from v1 storage instead. -func (manager *SafePointV2Manager) getGCSafePoint(keyspaceID uint32) (*endpoint.GCSafePointV2, error) { - v2SafePoint, err := manager.v2Storage.LoadGCSafePointV2(keyspaceID) - if err != nil { - return nil, err - } - // If failed to find a valid safe point, check if a safe point exist in v1 storage, and use it. - if v2SafePoint.SafePoint == 0 { - v1SafePoint, err := manager.v1Storage.LoadGCSafePoint() - if err != nil { - return nil, err - } - log.Info("keyspace does not have a gc safe point, using v1 gc safe point instead", - zap.Uint32("keyspace-id", keyspaceID), - zap.Uint64("gc-safe-point-v1", v1SafePoint)) - v2SafePoint.SafePoint = v1SafePoint - } - return v2SafePoint, nil -} - -// UpdateGCSafePoint is used to update gc safe point for given keyspace. -func (manager *SafePointV2Manager) UpdateGCSafePoint(gcSafePoint *endpoint.GCSafePointV2) (oldGCSafePoint *endpoint.GCSafePointV2, err error) { - manager.Lock(gcSafePoint.KeyspaceID) - defer manager.Unlock(gcSafePoint.KeyspaceID) - // Check if keyspace is valid to load. - if err = manager.checkKeyspace(gcSafePoint.KeyspaceID, true); err != nil { - return - } - oldGCSafePoint, err = manager.getGCSafePoint(gcSafePoint.KeyspaceID) - if err != nil { - return - } - if oldGCSafePoint.SafePoint >= gcSafePoint.SafePoint { - return - } - err = manager.v2Storage.SaveGCSafePointV2(gcSafePoint) - return -} - -// UpdateServiceSafePoint update keyspace service safe point with the given serviceSafePoint. -func (manager *SafePointV2Manager) UpdateServiceSafePoint(serviceSafePoint *endpoint.ServiceSafePointV2, now time.Time) (*endpoint.ServiceSafePointV2, error) { - manager.Lock(serviceSafePoint.KeyspaceID) - defer manager.Unlock(serviceSafePoint.KeyspaceID) - // Check if keyspace is valid to update. - if err := manager.checkKeyspace(serviceSafePoint.KeyspaceID, true); err != nil { - return nil, err - } - minServiceSafePoint, err := manager.v2Storage.LoadMinServiceSafePointV2(serviceSafePoint.KeyspaceID, now) - if err != nil { - return nil, err - } - if serviceSafePoint.SafePoint < minServiceSafePoint.SafePoint { - log.Warn("failed to update service safe point, proposed safe point smaller than current min", - zap.Error(err), - zap.Uint32("keyspace-id", serviceSafePoint.KeyspaceID), - zap.Uint64("request-service-safe-point", serviceSafePoint.SafePoint), - zap.Uint64("min-service-safe-point", minServiceSafePoint.SafePoint), - ) - return minServiceSafePoint, nil - } - if err = manager.v2Storage.SaveServiceSafePointV2(serviceSafePoint); err != nil { - return nil, err - } - // If the updated safe point is the original min safe point, reload min safe point. - if serviceSafePoint.ServiceID == minServiceSafePoint.ServiceID { - minServiceSafePoint, err = manager.v2Storage.LoadMinServiceSafePointV2(serviceSafePoint.KeyspaceID, now) - } - if err != nil { - log.Info("update service safe point", - zap.String("service-id", serviceSafePoint.ServiceID), - zap.Int64("expire-at", serviceSafePoint.ExpiredAt), - zap.Uint64("safepoint", serviceSafePoint.SafePoint), - ) - } - return minServiceSafePoint, err -} - -// RemoveServiceSafePoint remove keyspace service safe point with the given keyspaceID and serviceID. -func (manager *SafePointV2Manager) RemoveServiceSafePoint(keyspaceID uint32, serviceID string, now time.Time) (*endpoint.ServiceSafePointV2, error) { - manager.Lock(keyspaceID) - defer manager.Unlock(keyspaceID) - // Check if keyspace is valid to update. - if err := manager.checkKeyspace(keyspaceID, true); err != nil { - return nil, err - } - // Remove target safe point. - if err := manager.v2Storage.RemoveServiceSafePointV2(keyspaceID, serviceID); err != nil { - return nil, err - } - // Load min safe point. - minServiceSafePoint, err := manager.v2Storage.LoadMinServiceSafePointV2(keyspaceID, now) - if err != nil { - return nil, err - } - return minServiceSafePoint, nil -} diff --git a/pkg/schedule/checker/rule_checker_test.go b/pkg/schedule/checker/rule_checker_test.go index 029cf89e21a..67750acd7a1 100644 --- a/pkg/schedule/checker/rule_checker_test.go +++ b/pkg/schedule/checker/rule_checker_test.go @@ -2381,7 +2381,8 @@ func (suite *ruleCheckerTestSuite) TestFixBetterLocationEngineConstraint() { Count: 3, LocationLabels: []string{"zone", "host"}, } - suite.ruleManager.SetRule(rule) + err := suite.ruleManager.SetRule(rule) + re.NoError(err) region1 := suite.cluster.GetRegion(1) op := suite.rc.Check(region1) re.Empty(op) @@ -2402,7 +2403,8 @@ func (suite *ruleCheckerTestSuite) TestFixBetterLocationEngineConstraint() { LocationLabels: []string{"zone", "host"}, } - suite.ruleManager.SetRule(ruleTiFlash) + err = suite.ruleManager.SetRule(ruleTiFlash) + re.NoError(err) suite.cluster.AddRegionWithLearner(2, 1, []uint64{2, 3}, []uint64{5, 6, 7}) region2 := suite.cluster.GetRegion(2) op = suite.rc.Check(region2) diff --git a/pkg/storage/endpoint/gc_safe_point.go b/pkg/storage/endpoint/gc_safe_point.go deleted file mode 100644 index 6dd3c1bb6f1..00000000000 --- a/pkg/storage/endpoint/gc_safe_point.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package endpoint - -import ( - "encoding/json" - "math" - "strconv" - "time" - - clientv3 "go.etcd.io/etcd/client/v3" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - - "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/keyspace/constant" - "github.com/tikv/pd/pkg/utils/keypath" -) - -// WARNING: The content of this file is going to be deprecated and replaced by `gc_states.go`. - -// GCSafePointStorage defines the storage operations on the GC safe point. -type GCSafePointStorage interface { - LoadGCSafePoint() (uint64, error) - SaveGCSafePoint(safePoint uint64) error - LoadMinServiceGCSafePoint(now time.Time) (*ServiceSafePoint, error) - LoadAllServiceGCSafePoints() ([]*ServiceSafePoint, error) - SaveServiceGCSafePoint(ssp *ServiceSafePoint) error - RemoveServiceGCSafePoint(serviceID string) error -} - -var _ GCSafePointStorage = (*StorageEndpoint)(nil) - -// LoadGCSafePoint loads current GC safe point from storage. -func (se *StorageEndpoint) LoadGCSafePoint() (uint64, error) { - value, err := se.Load(keypath.GCSafePointPath(constant.NullKeyspaceID)) - if err != nil || value == "" { - return 0, err - } - safePoint, err := strconv.ParseUint(value, 16, 64) - if err != nil { - return 0, errs.ErrStrconvParseUint.Wrap(err).GenWithStackByArgs() - } - return safePoint, nil -} - -// SaveGCSafePoint saves new GC safe point to storage. -func (se *StorageEndpoint) SaveGCSafePoint(safePoint uint64) error { - value := strconv.FormatUint(safePoint, 16) - return se.Save(keypath.GCSafePointPath(constant.NullKeyspaceID), value) -} - -// LoadMinServiceGCSafePoint returns the minimum safepoint across all services -func (se *StorageEndpoint) LoadMinServiceGCSafePoint(now time.Time) (*ServiceSafePoint, error) { - prefix := keypath.ServiceGCSafePointPrefix() - prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - keys, values, err := se.LoadRange(prefix, prefixEnd, 0) - if err != nil { - return nil, err - } - if len(keys) == 0 { - // There's no service safepoint. It may be a new cluster, or upgraded from an older version where all service - // safepoints are missing. For the second case, we have no way to recover it. Store an initial value 0 for - // gc_worker. - return se.initServiceGCSafePointForGCWorker() - } - - hasGCWorker := false - min := &ServiceSafePoint{SafePoint: math.MaxUint64} - for i, key := range keys { - ssp := &ServiceSafePoint{} - if err := json.Unmarshal([]byte(values[i]), ssp); err != nil { - return nil, err - } - if ssp.ServiceID == keypath.GCWorkerServiceSafePointID { - hasGCWorker = true - // If gc_worker's expire time is incorrectly set, fix it. - if ssp.ExpiredAt != math.MaxInt64 { - ssp.ExpiredAt = math.MaxInt64 - err = se.SaveServiceGCSafePoint(ssp) - if err != nil { - return nil, errors.Trace(err) - } - } - } - - if ssp.ExpiredAt < now.Unix() { - if err := se.Remove(key); err != nil { - log.Error("failed to remove expired service safepoint", errs.ZapError(err)) - } - continue - } - if ssp.SafePoint < min.SafePoint { - min = ssp - } - } - - if min.SafePoint == math.MaxUint64 { - // There's no valid safepoints and we have no way to recover it. Just set gc_worker to 0. - log.Info("there are no valid service safepoints. init gc_worker's service safepoint to 0") - return se.initServiceGCSafePointForGCWorker() - } - - if !hasGCWorker { - // If there exists some service safepoints but gc_worker is missing, init it with the min value among all - // safepoints (including expired ones) - return se.initServiceGCSafePointForGCWorker() - } - - return min, nil -} - -func (se *StorageEndpoint) initServiceGCSafePointForGCWorker() (*ServiceSafePoint, error) { - // Temporary solution: - // Use the txn safe point as the initial value of gc_worker. - txnSafePoint, err := se.GetGCStateProvider().LoadTxnSafePoint(constant.NullKeyspaceID) - if err != nil { - return nil, err - } - ssp := &ServiceSafePoint{ - ServiceID: keypath.GCWorkerServiceSafePointID, - SafePoint: txnSafePoint, - ExpiredAt: math.MaxInt64, - } - if err := se.SaveServiceGCSafePoint(ssp); err != nil { - return nil, err - } - return ssp, nil -} - -// LoadAllServiceGCSafePoints returns all services GC safepoints -func (se *StorageEndpoint) LoadAllServiceGCSafePoints() ([]*ServiceSafePoint, error) { - prefix := keypath.ServiceGCSafePointPrefix() - prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - keys, values, err := se.LoadRange(prefix, prefixEnd, 0) - if err != nil { - return nil, err - } - if len(keys) == 0 { - return []*ServiceSafePoint{}, nil - } - - ssps := make([]*ServiceSafePoint, 0, len(keys)) - for i := range keys { - ssp := &ServiceSafePoint{} - if err := json.Unmarshal([]byte(values[i]), ssp); err != nil { - return nil, err - } - ssps = append(ssps, ssp) - } - - return ssps, nil -} - -// SaveServiceGCSafePoint saves a GC safepoint for the service -func (se *StorageEndpoint) SaveServiceGCSafePoint(ssp *ServiceSafePoint) error { - if ssp.ServiceID == "" { - return errors.New("service id of service safepoint cannot be empty") - } - - if ssp.ServiceID == keypath.GCWorkerServiceSafePointID && ssp.ExpiredAt != math.MaxInt64 { - return errors.New("TTL of gc_worker's service safe point must be infinity") - } - - return se.saveJSON(keypath.ServiceGCSafePointPath(ssp.ServiceID), ssp) -} - -// RemoveServiceGCSafePoint removes a GC safepoint for the service -func (se *StorageEndpoint) RemoveServiceGCSafePoint(serviceID string) error { - if serviceID == keypath.GCWorkerServiceSafePointID { - return errors.New("cannot remove service safe point of gc_worker") - } - key := keypath.ServiceGCSafePointPath(serviceID) - return se.Remove(key) -} diff --git a/pkg/storage/endpoint/gc_states.go b/pkg/storage/endpoint/gc_states.go index 954f463ba3b..b68a690564a 100644 --- a/pkg/storage/endpoint/gc_states.go +++ b/pkg/storage/endpoint/gc_states.go @@ -306,6 +306,7 @@ func (p GCStateProvider) LoadGCBarrier(keyspaceID uint32, barrierID string) (*GC } // LoadAllGCBarriers loads all GC barriers of the given keyspace. +// Note that reserved barrier IDs (e.g., "gc_worker") are not filtered out here. func (p GCStateProvider) LoadAllGCBarriers(keyspaceID uint32) ([]*GCBarrier, error) { prefix := keypath.GCBarrierPrefix(keyspaceID) // TODO: Limit the count for each call. @@ -422,8 +423,8 @@ func (p GCStateProvider) RunInGCStateTransaction(f func(wb *GCStateWriteBatch) e } // CompatibleLoadAllServiceGCSafePoints returns all services GC safe points with their etcd key. -func (p GCStateProvider) CompatibleLoadAllServiceGCSafePoints() ([]string, []*ServiceSafePoint, error) { - prefix := keypath.GCBarrierPrefix(constant.NullKeyspaceID) +func (p GCStateProvider) CompatibleLoadAllServiceGCSafePoints(keyspaceID uint32) ([]string, []*ServiceSafePoint, error) { + prefix := keypath.GCBarrierPrefix(keyspaceID) keys, ssps, err := loadJSONByPrefix[*ServiceSafePoint](p.storage, prefix, 0) if err != nil { return nil, nil, err diff --git a/pkg/storage/endpoint/gc_states_test.go b/pkg/storage/endpoint/gc_states_test.go index 53f3ad43147..315b3d380a2 100644 --- a/pkg/storage/endpoint/gc_states_test.go +++ b/pkg/storage/endpoint/gc_states_test.go @@ -480,7 +480,7 @@ func TestGCBarrier(t *testing.T) { if keyspaceID == constant.NullKeyspaceID { // Check by the legacy service safe point API for null keyspace. - keys, ssps, err := provider.CompatibleLoadAllServiceGCSafePoints() + keys, ssps, err := provider.CompatibleLoadAllServiceGCSafePoints(keyspaceID) re.NoError(err) re.Len(keys, 3) re.Len(ssps, 3) @@ -758,7 +758,7 @@ func TestDataPhysicalRepresentation(t *testing.T) { re.Equal("instance2", key) re.Equal(uint64(456139133457530888), minStartTS) - keys, ssps, err := provider.CompatibleLoadAllServiceGCSafePoints() + keys, ssps, err := provider.CompatibleLoadAllServiceGCSafePoints(constant.NullKeyspaceID) re.NoError(err) re.Equal([]string{ "/pd/0/gc/safe_point/service/gc_worker", diff --git a/pkg/storage/endpoint/safepoint_v2.go b/pkg/storage/endpoint/safepoint_v2.go deleted file mode 100644 index 13489831c78..00000000000 --- a/pkg/storage/endpoint/safepoint_v2.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright 2023 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package endpoint - -import ( - "encoding/json" - "math" - "time" - - clientv3 "go.etcd.io/etcd/client/v3" - "go.uber.org/zap" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - - "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/utils/keypath" -) - -// GCSafePointV2 represents the overall safe point for a specific keyspace. -type GCSafePointV2 struct { - KeyspaceID uint32 `json:"keyspace_id"` - SafePoint uint64 `json:"safe_point"` -} - -// ServiceSafePointV2 represents a service's safepoint under a specific keyspace. -// Services can post service safe point to prevent gc safe point from incrementing. -type ServiceSafePointV2 struct { - KeyspaceID uint32 `json:"keyspace_id"` - ServiceID string `json:"service_id"` - ExpiredAt int64 `json:"expired_at"` - SafePoint uint64 `json:"safe_point"` -} - -// SafePointV2Storage defines the storage operations on safe point v2. -type SafePointV2Storage interface { - LoadGCSafePointV2(keyspaceID uint32) (*GCSafePointV2, error) - SaveGCSafePointV2(gcSafePoint *GCSafePointV2) error - LoadAllGCSafePoints() ([]*GCSafePointV2, error) - - LoadMinServiceSafePointV2(keyspaceID uint32, now time.Time) (*ServiceSafePointV2, error) - LoadServiceSafePointV2(keyspaceID uint32, serviceID string) (*ServiceSafePointV2, error) - - SaveServiceSafePointV2(serviceSafePoint *ServiceSafePointV2) error - RemoveServiceSafePointV2(keyspaceID uint32, serviceID string) error -} - -var _ SafePointV2Storage = (*StorageEndpoint)(nil) - -// LoadGCSafePointV2 loads gc safe point for the given keyspace. -func (se *StorageEndpoint) LoadGCSafePointV2(keyspaceID uint32) (*GCSafePointV2, error) { - key := keypath.GCSafePointV2Path(keyspaceID) - value, err := se.Load(key) - if err != nil { - return nil, err - } - // GC Safe Point does not exist for the given keyspace - if value == "" { - return &GCSafePointV2{ - KeyspaceID: keyspaceID, - SafePoint: 0, - }, nil - } - gcSafePoint := &GCSafePointV2{} - if err = json.Unmarshal([]byte(value), gcSafePoint); err != nil { - return nil, errs.ErrJSONUnmarshal.Wrap(err).GenWithStackByCause() - } - return gcSafePoint, nil -} - -// SaveGCSafePointV2 saves gc safe point for the given keyspace. -func (se *StorageEndpoint) SaveGCSafePointV2(gcSafePoint *GCSafePointV2) error { - return se.saveJSON(keypath.GCSafePointV2Path(gcSafePoint.KeyspaceID), gcSafePoint) -} - -// LoadAllGCSafePoints returns gc safe point for all keyspaces -func (se *StorageEndpoint) LoadAllGCSafePoints() ([]*GCSafePointV2, error) { - prefix := keypath.GCSafePointV2Prefix() - prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - _, values, err := se.LoadRange(prefix, prefixEnd, 0) - if err != nil { - return nil, err - } - gcSafePoints := make([]*GCSafePointV2, 0, len(values)) - for _, value := range values { - gcSafePoint := &GCSafePointV2{} - if err = json.Unmarshal([]byte(value), gcSafePoint); err != nil { - return nil, errs.ErrJSONUnmarshal.Wrap(err).GenWithStackByCause() - } - gcSafePoints = append(gcSafePoints, gcSafePoint) - } - return gcSafePoints, nil -} - -// LoadMinServiceSafePointV2 returns the minimum safepoint for the given keyspace. -// If no service safe point exist for the given key space or all the service safe points just expired, return nil. -// This also attempt to remove expired service safe point. -func (se *StorageEndpoint) LoadMinServiceSafePointV2(keyspaceID uint32, now time.Time) (*ServiceSafePointV2, error) { - prefix := keypath.ServiceSafePointV2Prefix(keyspaceID) - prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - keys, values, err := se.LoadRange(prefix, prefixEnd, 0) - if err != nil { - return nil, err - } - if len(keys) == 0 { - return se.initServiceSafePointV2ForGCWorker(keyspaceID) - } - - hasGCWorker := false - min := &ServiceSafePointV2{KeyspaceID: keyspaceID, SafePoint: math.MaxUint64} - for i, key := range keys { - serviceSafePoint := &ServiceSafePointV2{} - if err = json.Unmarshal([]byte(values[i]), serviceSafePoint); err != nil { - return nil, err - } - if serviceSafePoint.ServiceID == keypath.GCWorkerServiceSafePointID { - hasGCWorker = true - // If gc_worker's expire time is incorrectly set, fix it. - if serviceSafePoint.ExpiredAt != math.MaxInt64 { - serviceSafePoint.ExpiredAt = math.MaxInt64 - err = se.SaveServiceSafePointV2(serviceSafePoint) - if err != nil { - return nil, errors.Trace(err) - } - } - } - if serviceSafePoint.ExpiredAt < now.Unix() { - if err = se.Remove(key); err != nil { - log.Warn("failed to remove expired service safe point", zap.Error(err)) - } - continue - } - if serviceSafePoint.SafePoint < min.SafePoint { - min = serviceSafePoint - } - } - if min.SafePoint == math.MaxUint64 { - // No service safe point or all of them are expired, set min service safe point to 0 to allow any update - log.Info("there are no valid service safepoints. init gc_worker's service safepoint to 0") - return se.initServiceSafePointV2ForGCWorker(keyspaceID) - } - if !hasGCWorker { - // If there exists some service safepoints but gc_worker is missing, init it with the min value among all - // safepoints (including expired ones) - return se.initServiceSafePointV2ForGCWorker(keyspaceID) - } - return min, nil -} - -// LoadServiceSafePointV2 returns ServiceSafePointV2 for given keyspaceID and serviceID. -func (se *StorageEndpoint) LoadServiceSafePointV2(keyspaceID uint32, serviceID string) (*ServiceSafePointV2, error) { - key := keypath.ServiceSafePointV2Path(keyspaceID, serviceID) - value, err := se.Load(key) - if err != nil { - return nil, err - } - // Service Safe Point does not exist for the given keyspaceID and serviceID - if value == "" { - return nil, nil - } - serviceSafePoint := &ServiceSafePointV2{} - if err = json.Unmarshal([]byte(value), serviceSafePoint); err != nil { - return nil, err - } - return serviceSafePoint, nil -} - -func (se *StorageEndpoint) initServiceSafePointV2ForGCWorker(keyspaceID uint32) (*ServiceSafePointV2, error) { - // Temporary solution: - // Use the txn safe point as the initial value of gc_worker. - txnSafePoint, err := se.GetGCStateProvider().LoadTxnSafePoint(keyspaceID) - if err != nil { - return nil, err - } - ssp := &ServiceSafePointV2{ - KeyspaceID: keyspaceID, - ServiceID: keypath.GCWorkerServiceSafePointID, - SafePoint: txnSafePoint, - ExpiredAt: math.MaxInt64, - } - if err := se.SaveServiceSafePointV2(ssp); err != nil { - return nil, err - } - return ssp, nil -} - -// SaveServiceSafePointV2 stores service safe point to etcd. -func (se *StorageEndpoint) SaveServiceSafePointV2(serviceSafePoint *ServiceSafePointV2) error { - if serviceSafePoint.ServiceID == "" { - return errors.New("service id of service safepoint cannot be empty") - } - - if serviceSafePoint.ServiceID == keypath.GCWorkerServiceSafePointID && serviceSafePoint.ExpiredAt != math.MaxInt64 { - return errors.New("TTL of gc_worker's service safe point must be infinity") - } - - key := keypath.ServiceSafePointV2Path(serviceSafePoint.KeyspaceID, serviceSafePoint.ServiceID) - return se.saveJSON(key, serviceSafePoint) -} - -// RemoveServiceSafePointV2 removes a service safe point. -func (se *StorageEndpoint) RemoveServiceSafePointV2(keyspaceID uint32, serviceID string) error { - if serviceID == keypath.GCWorkerServiceSafePointID { - return errors.New("cannot remove service safe point of gc_worker") - } - key := keypath.ServiceSafePointV2Path(keyspaceID, serviceID) - return se.Remove(key) -} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 3ff0a8c1d2e..4a37558ed02 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -39,11 +39,9 @@ type Storage interface { endpoint.MetaStorage endpoint.RuleStorage endpoint.ReplicationStatusStorage - endpoint.GCSafePointStorage endpoint.GCStateStorage endpoint.MinResolvedTSStorage endpoint.ExternalTSStorage - endpoint.SafePointV2Storage endpoint.KeyspaceStorage endpoint.ResourceGroupStorage endpoint.TSOStorage diff --git a/pkg/storage/storage_gc_test.go b/pkg/storage/storage_gc_test.go deleted file mode 100644 index 3f63dfa21fb..00000000000 --- a/pkg/storage/storage_gc_test.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "math" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/tikv/pd/pkg/storage/endpoint" - "github.com/tikv/pd/pkg/utils/etcdutil" - "github.com/tikv/pd/pkg/utils/keypath" -) - -func testGCSafePoints() []*endpoint.GCSafePointV2 { - gcSafePoint := []*endpoint.GCSafePointV2{ - {KeyspaceID: uint32(1), SafePoint: 0}, - {KeyspaceID: uint32(2), SafePoint: 1}, - {KeyspaceID: uint32(3), SafePoint: 4396}, - {KeyspaceID: uint32(4), SafePoint: 23333333333}, - {KeyspaceID: uint32(5), SafePoint: math.MaxUint64}, - } - - return gcSafePoint -} - -func testServiceSafePoints() []*endpoint.ServiceSafePointV2 { - expireAt := time.Now().Add(100 * time.Second).Unix() - serviceSafePoints := []*endpoint.ServiceSafePointV2{ - {KeyspaceID: uint32(1), ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, - {KeyspaceID: uint32(1), ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, - {KeyspaceID: uint32(1), ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, - {KeyspaceID: uint32(2), ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, - {KeyspaceID: uint32(2), ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, - {KeyspaceID: uint32(2), ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, - {KeyspaceID: uint32(3), ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, - {KeyspaceID: uint32(3), ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, - {KeyspaceID: uint32(3), ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, - } - return serviceSafePoints -} - -func TestSaveLoadServiceSafePoint(t *testing.T) { - re := require.New(t) - storage := NewStorageWithMemoryBackend() - testServiceSafepoints := testServiceSafePoints() - for i := range testServiceSafepoints { - re.NoError(storage.SaveServiceSafePointV2(testServiceSafepoints[i])) - } - for i := range testServiceSafepoints { - loadedServiceSafePoint, err := storage.LoadServiceSafePointV2(testServiceSafepoints[i].KeyspaceID, testServiceSafepoints[i].ServiceID) - re.NoError(err) - re.Equal(testServiceSafepoints[i].SafePoint, loadedServiceSafePoint.SafePoint) - } -} - -func TestLoadMinServiceSafePoint(t *testing.T) { - re := require.New(t) - _, client, clean := etcdutil.NewTestEtcdCluster(t, 1) - defer clean() - storage := NewStorageWithEtcdBackend(client) - currentTime := time.Now() - expireAt1 := currentTime.Add(1000 * time.Second).Unix() - expireAt2 := currentTime.Add(2000 * time.Second).Unix() - expireAt3 := currentTime.Add(3000 * time.Second).Unix() - - testKeyspaceID := uint32(1) - serviceSafePoints := []*endpoint.ServiceSafePointV2{ - {KeyspaceID: testKeyspaceID, ServiceID: "0", ExpiredAt: expireAt1, SafePoint: 300}, - {KeyspaceID: testKeyspaceID, ServiceID: "1", ExpiredAt: expireAt2, SafePoint: 400}, - {KeyspaceID: testKeyspaceID, ServiceID: "2", ExpiredAt: expireAt3, SafePoint: 500}, - } - - err := storage.GetGCStateProvider().RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error { - return wb.SetTxnSafePoint(testKeyspaceID, 100) - }) - re.NoError(err) - - for _, serviceSafePoint := range serviceSafePoints { - re.NoError(storage.SaveServiceSafePointV2(serviceSafePoint)) - } - minSafePoint, err := storage.LoadMinServiceSafePointV2(testKeyspaceID, currentTime) - re.NoError(err) - re.Equal(uint64(100), minSafePoint.SafePoint) - - // gc_worker service safepoint will not be removed. - ssp, err := storage.LoadMinServiceSafePointV2(testKeyspaceID, currentTime.Add(5000*time.Second)) - re.NoError(err) - re.Equal(keypath.GCWorkerServiceSafePointID, ssp.ServiceID) -} - -func TestRemoveServiceSafePoint(t *testing.T) { - re := require.New(t) - storage := NewStorageWithMemoryBackend() - testServiceSafepoint := testServiceSafePoints() - // save service safe points - for _, serviceSafePoint := range testServiceSafepoint { - re.NoError(storage.SaveServiceSafePointV2(serviceSafePoint)) - } - // remove saved service safe points - for _, serviceSafePoint := range testServiceSafepoint { - re.NoError(storage.RemoveServiceSafePointV2(serviceSafePoint.KeyspaceID, serviceSafePoint.ServiceID)) - } - // check that service safe points are empty - for i := range testServiceSafepoint { - loadedSafePoint, err := storage.LoadServiceSafePointV2(testServiceSafepoint[i].KeyspaceID, testServiceSafepoint[i].ServiceID) - re.NoError(err) - re.Nil(loadedSafePoint) - } -} - -func TestSaveLoadGCSafePoint(t *testing.T) { - re := require.New(t) - storage := NewStorageWithMemoryBackend() - testGCSafePoints := testGCSafePoints() - for _, testGCSafePoint := range testGCSafePoints { - testSpaceID := testGCSafePoint.KeyspaceID - testSafePoint := testGCSafePoint.SafePoint - err := storage.SaveGCSafePointV2(testGCSafePoint) - re.NoError(err) - loadGCSafePoint, err := storage.LoadGCSafePointV2(testSpaceID) - re.NoError(err) - re.Equal(testSafePoint, loadGCSafePoint.SafePoint) - } - - _, err2 := storage.LoadGCSafePointV2(999) - re.NoError(err2) -} - -func TestLoadEmpty(t *testing.T) { - re := require.New(t) - storage := NewStorageWithMemoryBackend() - - // loading non-existing GC safepoint should return 0 - gcSafePoint, err := storage.LoadGCSafePointV2(1) - re.NoError(err) - re.Equal(uint64(0), gcSafePoint.SafePoint) - - // loading non-existing service safepoint should return nil - serviceSafePoint, err := storage.LoadServiceSafePointV2(1, "testService") - re.NoError(err) - re.Nil(serviceSafePoint) -} diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 9b64057b4af..50430b15f24 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -16,17 +16,12 @@ package storage import ( "context" - "encoding/json" "fmt" - "math" "math/rand" "sort" - "strings" "testing" - "time" "github.com/stretchr/testify/require" - clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/goleak" "github.com/pingcap/failpoint" @@ -135,90 +130,6 @@ func TestStoreWeight(t *testing.T) { } } -func TestLoadGCSafePoint(t *testing.T) { - re := require.New(t) - storage := NewStorageWithMemoryBackend() - testData := []uint64{0, 1, 2, 233, 2333, 23333333333, math.MaxUint64} - - r, e := storage.LoadGCSafePoint() - re.Equal(uint64(0), r) - re.NoError(e) - for _, safePoint := range testData { - err := storage.SaveGCSafePoint(safePoint) - re.NoError(err) - safePoint1, err := storage.LoadGCSafePoint() - re.NoError(err) - re.Equal(safePoint1, safePoint) - } -} - -func TestSaveServiceGCSafePoint(t *testing.T) { - re := require.New(t) - storage := NewStorageWithMemoryBackend() - expireAt := time.Now().Add(100 * time.Second).Unix() - serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "3", ExpiredAt: expireAt, SafePoint: 3}, - } - - for _, ssp := range serviceSafePoints { - re.NoError(storage.SaveServiceGCSafePoint(ssp)) - } - - prefix := keypath.ServiceGCSafePointPrefix() - prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - keys, values, err := storage.LoadRange(prefix, prefixEnd, len(serviceSafePoints)) - re.NoError(err) - re.Len(keys, 3) - re.Len(values, 3) - - ssp := &endpoint.ServiceSafePoint{} - for i, key := range keys { - re.True(strings.HasSuffix(key, serviceSafePoints[i].ServiceID)) - - re.NoError(json.Unmarshal([]byte(values[i]), ssp)) - re.Equal(serviceSafePoints[i].ServiceID, ssp.ServiceID) - re.Equal(serviceSafePoints[i].ExpiredAt, ssp.ExpiredAt) - re.Equal(serviceSafePoints[i].SafePoint, ssp.SafePoint) - } -} - -func TestLoadMinServiceGCSafePoint(t *testing.T) { - re := require.New(t) - storage := NewStorageWithMemoryBackend() - expireAt := time.Now().Add(1000 * time.Second).Unix() - serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "1", ExpiredAt: 0, SafePoint: 1}, - {ServiceID: "2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "3", ExpiredAt: expireAt, SafePoint: 3}, - } - - for _, ssp := range serviceSafePoints { - re.NoError(storage.SaveServiceGCSafePoint(ssp)) - } - - // gc_worker's service safe point will be automatically inserted with initial value 0. If txn safe point was set, - // the initial value of gc_worker's service safe point would be the same value as the txn safe point; however we - // didn't. - ssp, err := storage.LoadMinServiceGCSafePoint(time.Now()) - re.NoError(err) - re.Equal(uint64(0), ssp.SafePoint) - - // Advance gc_worker's safepoint - re.NoError(storage.SaveServiceGCSafePoint(&endpoint.ServiceSafePoint{ - ServiceID: "gc_worker", - ExpiredAt: math.MaxInt64, - SafePoint: 10, - })) - - ssp, err = storage.LoadMinServiceGCSafePoint(time.Now()) - re.NoError(err) - re.Equal("2", ssp.ServiceID) - re.Equal(expireAt, ssp.ExpiredAt) - re.Equal(uint64(2), ssp.SafePoint) -} - func TestTryGetLocalRegionStorage(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) diff --git a/server/api/service_gc_safepoint.go b/server/api/service_gc_safepoint.go index a86e4bb35db..658775a080a 100644 --- a/server/api/service_gc_safepoint.go +++ b/server/api/service_gc_safepoint.go @@ -21,6 +21,7 @@ import ( "github.com/gorilla/mux" "github.com/unrolled/render" + "github.com/tikv/pd/pkg/keyspace/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/server" ) @@ -54,17 +55,18 @@ type ListServiceGCSafepoint struct { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /gc/safepoint [get] func (h *serviceGCSafepointHandler) GetGCSafePoint(w http.ResponseWriter, _ *http.Request) { - storage := h.svr.GetStorage() - gcSafepoint, err := storage.LoadGCSafePoint() + gcStateManager := h.svr.GetGCStateManager() + gcState, err := gcStateManager.GetGCState(constant.NullKeyspaceID) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - ssps, err := storage.LoadAllServiceGCSafePoints() - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return + + ssps := make([]*endpoint.ServiceSafePoint, 0, len(gcState.GCBarriers)) + for _, barrier := range gcState.GCBarriers { + ssps = append(ssps, barrier.ToServiceSafePoint(constant.NullKeyspaceID)) } + var minSSp *endpoint.ServiceSafePoint for _, ssp := range ssps { if (minSSp == nil || minSSp.SafePoint > ssp.SafePoint) && @@ -77,7 +79,7 @@ func (h *serviceGCSafepointHandler) GetGCSafePoint(w http.ResponseWriter, _ *htt minServiceGcSafepoint = minSSp.SafePoint } list := ListServiceGCSafepoint{ - GCSafePoint: gcSafepoint, + GCSafePoint: gcState.GCSafePoint, ServiceGCSafepoints: ssps, MinServiceGcSafepoint: minServiceGcSafepoint, } @@ -94,9 +96,16 @@ func (h *serviceGCSafepointHandler) GetGCSafePoint(w http.ResponseWriter, _ *htt // @Router /gc/safepoint/{service_id} [delete] // @Tags rule func (h *serviceGCSafepointHandler) DeleteGCSafePoint(w http.ResponseWriter, r *http.Request) { - storage := h.svr.GetStorage() + // Directly write to the storage and bypassing the existing constraint checks. + // It's risky to do this, but when this HTTP API is used, it usually means that we are already taking risks. + provider := h.svr.GetStorage().GetGCStateProvider() serviceID := mux.Vars(r)["service_id"] - err := storage.RemoveServiceGCSafePoint(serviceID) + err := provider.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error { + // As GC barriers and service safe points shares the same data, deleting GC barriers acts the same as deleting + // service safe points. + err := wb.DeleteGCBarrier(constant.NullKeyspaceID, serviceID) + return err + }) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index ab12d1bdcb8..21157e61f66 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -42,7 +42,6 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/gc" "github.com/tikv/pd/pkg/gctuner" "github.com/tikv/pd/pkg/id" "github.com/tikv/pd/pkg/keyspace" @@ -135,7 +134,6 @@ type Server interface { ReplicateFileToMember(ctx context.Context, member *pdpb.Member, name string, data []byte) error GetKeyspaceGroupManager() *keyspace.GroupManager IsKeyspaceGroupEnabled() bool - GetSafePointV2Manager() *gc.SafePointV2Manager } // RaftCluster is used for cluster config management. diff --git a/server/gc_service.go b/server/gc_service.go index 3d13c2d934d..4cc0c83d2e0 100644 --- a/server/gc_service.go +++ b/server/gc_service.go @@ -16,30 +16,219 @@ package server import ( "context" - "encoding/json" - "fmt" "math" "time" - clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" - "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/gc" "github.com/tikv/pd/pkg/keyspace/constant" "github.com/tikv/pd/pkg/storage/endpoint" - "github.com/tikv/pd/pkg/utils/etcdutil" - "github.com/tikv/pd/pkg/utils/keypath" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" ) +// UpdateGCSafePoint implements gRPC PDServer. +// Deprecated: Use AdvanceGCSafePoint instead. Note that it's only for use of GC internal. +// +//nolint:staticcheck +func (s *GrpcServer) UpdateGCSafePoint(ctx context.Context, request *pdpb.UpdateGCSafePointRequest) (resp *pdpb.UpdateGCSafePointResponse, errRet error) { + log.Warn("deprecated API UpdateGCSafePoint is called", zap.Uint64("req-safe-point", request.GetSafePoint())) + defer func() { + if errRet != nil { + log.Error("deprecated API UpdateGCSafePoint encountered error", zap.Uint64("req-safe-point", request.GetSafePoint()), zap.Error(errRet)) + } else { + log.Warn("deprecated API UpdateGCSafePoint returned", zap.Uint64("req-safe-point", request.GetSafePoint()), zap.Uint64("resp-new-safe-point", resp.GetNewSafePoint())) + } + }() + + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() + } + fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { + return pdpb.NewPDClient(client).UpdateGCSafePoint(ctx, request) + } + if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + return nil, err + } else if rsp != nil { + return rsp.(*pdpb.UpdateGCSafePointResponse), err + } + + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.UpdateGCSafePointResponse{Header: notBootstrappedHeader()}, nil + } + + newSafePoint := request.GetSafePoint() + oldSafePoint, _, err := s.gcStateManager.CompatibleUpdateGCSafePoint(constant.NullKeyspaceID, newSafePoint) + if err != nil { + return nil, err + } + + if newSafePoint > oldSafePoint { + log.Info("updated gc safe point", + zap.Uint64("safe-point", newSafePoint)) + } else if newSafePoint < oldSafePoint { + log.Warn("trying to update gc safe point", + zap.Uint64("old-safe-point", oldSafePoint), + zap.Uint64("new-safe-point", newSafePoint)) + newSafePoint = oldSafePoint + } + + return &pdpb.UpdateGCSafePointResponse{ + Header: wrapHeader(), + NewSafePoint: newSafePoint, + }, nil +} + +// GetGCSafePoint implements gRPC PDServer. +// Deprecated: Use GetGCState instead. +// +//nolint:staticcheck +func (s *GrpcServer) GetGCSafePoint(ctx context.Context, request *pdpb.GetGCSafePointRequest) (resp *pdpb.GetGCSafePointResponse, errRet error) { + log.Warn("deprecated API GetGCSafePoint is called") + defer func() { + if errRet != nil { + log.Error("deprecated API GetGCSafePoint encountered error", zap.Error(errRet)) + } else { + log.Warn("deprecated API GetGCSafePoint returned", zap.Uint64("resp-safe-point", resp.GetSafePoint())) + } + }() + + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() + } + fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { + return pdpb.NewPDClient(client).GetGCSafePoint(ctx, request) + } + if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + return nil, err + } else if rsp != nil { + return rsp.(*pdpb.GetGCSafePointResponse), err + } + + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.GetGCSafePointResponse{Header: notBootstrappedHeader()}, nil + } + + safePoint, err := s.gcStateManager.CompatibleLoadGCSafePoint(constant.NullKeyspaceID) + if err != nil { + return nil, err + } + + return &pdpb.GetGCSafePointResponse{ + Header: wrapHeader(), + SafePoint: safePoint, + }, nil +} + +// UpdateServiceGCSafePoint update the safepoint for specific service +// Deprecated: Use SetGCBarrier instead. +// +//nolint:staticcheck +func (s *GrpcServer) UpdateServiceGCSafePoint(ctx context.Context, request *pdpb.UpdateServiceGCSafePointRequest) (resp *pdpb.UpdateServiceGCSafePointResponse, errRet error) { + log.Warn("deprecated API UpdateServiceGCSafePoint is called", + zap.String("req-service-id", string(request.GetServiceId())), + zap.Int64("req-ttl", request.GetTTL()), + zap.Uint64("req-safe-point", request.GetSafePoint())) + defer func() { + if errRet != nil { + log.Error("deprecated API UpdateServiceGCSafePoint encountered error", + zap.String("req-service-id", string(request.GetServiceId())), + zap.Int64("req-ttl", request.GetTTL()), + zap.Uint64("req-safe-point", request.GetSafePoint()), + zap.Error(errRet)) + } else { + log.Warn("deprecated API UpdateServiceGCSafePoint returned", + zap.String("req-service-id", string(request.GetServiceId())), + zap.Int64("req-ttl", request.GetTTL()), + zap.Uint64("req-safe-point", request.GetSafePoint()), + zap.String("resp-service-id", string(resp.GetServiceId())), + zap.Int64("resp-ttl", resp.GetTTL()), + zap.Uint64("resp-min-safe-point", resp.GetMinSafePoint())) + } + }() + + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() + } + fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { + return pdpb.NewPDClient(client).UpdateServiceGCSafePoint(ctx, request) + } + if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + return nil, err + } else if rsp != nil { + return rsp.(*pdpb.UpdateServiceGCSafePointResponse), err + } + + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.UpdateServiceGCSafePointResponse{Header: notBootstrappedHeader()}, nil + } + nowTSO, err := s.getGlobalTSO(ctx) + if err != nil { + return nil, err + } + now, _ := tsoutil.ParseTimestamp(nowTSO) + serviceID := string(request.ServiceId) + min, updated, err := s.gcStateManager.CompatibleUpdateServiceGCSafePoint(constant.NullKeyspaceID, serviceID, request.GetSafePoint(), request.GetTTL(), now) + if err != nil { + return nil, err + } + if updated { + log.Info("update service GC safe point", + zap.String("service-id", serviceID), + zap.Int64("expire-at", now.Unix()+request.GetTTL()), + zap.Uint64("safepoint", request.GetSafePoint())) + } + return &pdpb.UpdateServiceGCSafePointResponse{ + Header: wrapHeader(), + ServiceId: []byte(min.ServiceID), + TTL: min.ExpiredAt - now.Unix(), + MinSafePoint: min.SafePoint, + }, nil +} + // GetGCSafePointV2 return gc safe point for the given keyspace. -func (s *GrpcServer) GetGCSafePointV2(ctx context.Context, request *pdpb.GetGCSafePointV2Request) (*pdpb.GetGCSafePointV2Response, error) { +// Deprecated: Use GetGCState instead +// +//nolint:staticcheck +func (s *GrpcServer) GetGCSafePointV2(ctx context.Context, request *pdpb.GetGCSafePointV2Request) (resp *pdpb.GetGCSafePointV2Response, errRet error) { + log.Warn("deprecated API GetGCSafePointV2 is called", zap.Uint32("keyspace-id", request.GetKeyspaceId())) + defer func() { + if errRet != nil { + log.Error("deprecated API GetGCSafePointV2 encountered error", zap.Uint32("keyspace-id", request.GetKeyspaceId()), zap.Error(errRet)) + } else { + log.Warn("deprecated API GetGCSafePointV2 returned", zap.Uint32("keyspace-id", request.GetKeyspaceId()), zap.Uint64("resp-safe-point", resp.GetSafePoint())) + } + }() + + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() + } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetGCSafePointV2(ctx, request) } @@ -49,7 +238,12 @@ func (s *GrpcServer) GetGCSafePointV2(ctx context.Context, request *pdpb.GetGCSa return rsp.(*pdpb.GetGCSafePointV2Response), err } - safePoint, err := s.safePointV2Manager.LoadGCSafePoint(request.GetKeyspaceId()) + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.GetGCSafePointV2Response{Header: notBootstrappedHeader()}, nil + } + + safePoint, err := s.gcStateManager.CompatibleLoadGCSafePoint(request.GetKeyspaceId()) if err != nil { return &pdpb.GetGCSafePointV2Response{ @@ -59,12 +253,31 @@ func (s *GrpcServer) GetGCSafePointV2(ctx context.Context, request *pdpb.GetGCSa return &pdpb.GetGCSafePointV2Response{ Header: wrapHeader(), - SafePoint: safePoint.SafePoint, + SafePoint: safePoint, }, nil } // UpdateGCSafePointV2 update gc safe point for the given keyspace. -func (s *GrpcServer) UpdateGCSafePointV2(ctx context.Context, request *pdpb.UpdateGCSafePointV2Request) (*pdpb.UpdateGCSafePointV2Response, error) { +// Deprecated: Use AdvanceGCSafePoint instead. Note that it's only for use of GC internal. +// +//nolint:staticcheck +func (s *GrpcServer) UpdateGCSafePointV2(ctx context.Context, request *pdpb.UpdateGCSafePointV2Request) (resp *pdpb.UpdateGCSafePointV2Response, errRet error) { + log.Warn("deprecated API UpdateGCSafePointV2 is called", zap.Uint32("keyspace-id", request.GetKeyspaceId()), zap.Uint64("req-safe-point", request.GetSafePoint())) + defer func() { + if errRet != nil { + log.Error("deprecated API UpdateGCSafePointV2 encountered error", zap.Uint32("keyspace-id", request.GetKeyspaceId()), zap.Uint64("req-safe-point", request.GetSafePoint()), zap.Error(errRet)) + } else { + log.Warn("deprecated API UpdateGCSafePointV2 returned", zap.Uint32("keyspace-id", request.GetKeyspaceId()), zap.Uint64("req-safe-point", request.GetSafePoint()), zap.Uint64("resp-new-safe-point", resp.GetNewSafePoint())) + } + }() + + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() + } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).UpdateGCSafePointV2(ctx, request) } @@ -74,24 +287,27 @@ func (s *GrpcServer) UpdateGCSafePointV2(ctx context.Context, request *pdpb.Upda return rsp.(*pdpb.UpdateGCSafePointV2Response), err } + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.UpdateGCSafePointV2Response{Header: notBootstrappedHeader()}, nil + } + newSafePoint := request.GetSafePoint() - oldSafePoint, err := s.safePointV2Manager.UpdateGCSafePoint(&endpoint.GCSafePointV2{ - KeyspaceID: request.KeyspaceId, - SafePoint: request.SafePoint, - }) + oldSafePoint, _, err := s.gcStateManager.CompatibleUpdateGCSafePoint(request.GetKeyspaceId(), newSafePoint) if err != nil { return nil, err } - if newSafePoint > oldSafePoint.SafePoint { + + if newSafePoint > oldSafePoint { log.Info("updated gc safe point", zap.Uint64("safe-point", newSafePoint), zap.Uint32("keyspace-id", request.GetKeyspaceId())) - } else if newSafePoint < oldSafePoint.SafePoint { + } else if newSafePoint < oldSafePoint { log.Warn("trying to update gc safe point", - zap.Uint64("old-safe-point", oldSafePoint.SafePoint), + zap.Uint64("old-safe-point", oldSafePoint), zap.Uint64("new-safe-point", newSafePoint), zap.Uint32("keyspace-id", request.GetKeyspaceId())) - newSafePoint = oldSafePoint.SafePoint + newSafePoint = oldSafePoint } return &pdpb.UpdateGCSafePointV2Response{ @@ -101,7 +317,42 @@ func (s *GrpcServer) UpdateGCSafePointV2(ctx context.Context, request *pdpb.Upda } // UpdateServiceSafePointV2 update service safe point for the given keyspace. -func (s *GrpcServer) UpdateServiceSafePointV2(ctx context.Context, request *pdpb.UpdateServiceSafePointV2Request) (*pdpb.UpdateServiceSafePointV2Response, error) { +// Deprecated: Use SetGCBarrier instead. +// +//nolint:staticcheck +func (s *GrpcServer) UpdateServiceSafePointV2(ctx context.Context, request *pdpb.UpdateServiceSafePointV2Request) (resp *pdpb.UpdateServiceSafePointV2Response, errRet error) { + log.Warn("deprecated API UpdateServiceSafePointV2 is called", + zap.Uint32("keyspace-id", request.GetKeyspaceId()), + zap.String("req-service-id", string(request.GetServiceId())), + zap.Int64("req-ttl", request.GetTtl()), + zap.Uint64("req-safe-point", request.GetSafePoint())) + defer func() { + if errRet != nil { + log.Error("deprecated API UpdateServiceSafePointV2 encountered error", + zap.Uint32("keyspace-id", request.GetKeyspaceId()), + zap.String("req-service-id", string(request.GetServiceId())), + zap.Int64("req-ttl", request.GetTtl()), + zap.Uint64("req-safe-point", request.GetSafePoint()), + zap.Error(errRet)) + } else { + log.Warn("deprecated API UpdateServiceSafePointV2 returned", + zap.Uint32("keyspace-id", request.GetKeyspaceId()), + zap.String("req-service-id", string(request.GetServiceId())), + zap.Int64("req-ttl", request.GetTtl()), + zap.Uint64("req-safe-point", request.GetSafePoint()), + zap.String("resp-service-id", string(resp.GetServiceId())), + zap.Int64("resp-ttl", resp.GetTtl()), + zap.Uint64("resp-min-safe-point", resp.GetMinSafePoint())) + } + }() + + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() + } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).UpdateServiceSafePointV2(ctx, request) } @@ -111,94 +362,61 @@ func (s *GrpcServer) UpdateServiceSafePointV2(ctx context.Context, request *pdpb return rsp.(*pdpb.UpdateServiceSafePointV2Response), err } + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.UpdateServiceSafePointV2Response{Header: notBootstrappedHeader()}, nil + } + nowTSO, err := s.getGlobalTSO(ctx) if err != nil { return nil, err } now, _ := tsoutil.ParseTimestamp(nowTSO) - var minServiceSafePoint *endpoint.ServiceSafePointV2 - if request.Ttl < 0 { - minServiceSafePoint, err = s.safePointV2Manager.RemoveServiceSafePoint(request.GetKeyspaceId(), string(request.GetServiceId()), now) - } else { - serviceSafePoint := &endpoint.ServiceSafePointV2{ - KeyspaceID: request.GetKeyspaceId(), - ServiceID: string(request.GetServiceId()), - ExpiredAt: now.Unix() + request.GetTtl(), - SafePoint: request.GetSafePoint(), - } - // Fix possible overflow. - if math.MaxInt64-now.Unix() <= request.GetTtl() { - serviceSafePoint.ExpiredAt = math.MaxInt64 - } - minServiceSafePoint, err = s.safePointV2Manager.UpdateServiceSafePoint(serviceSafePoint, now) - } + serviceID := string(request.ServiceId) + min, _, err := s.gcStateManager.CompatibleUpdateServiceGCSafePoint(request.GetKeyspaceId(), serviceID, request.GetSafePoint(), request.GetTtl(), now) + if err != nil { return nil, err } return &pdpb.UpdateServiceSafePointV2Response{ Header: wrapHeader(), - ServiceId: []byte(minServiceSafePoint.ServiceID), - Ttl: minServiceSafePoint.ExpiredAt - now.Unix(), - MinSafePoint: minServiceSafePoint.SafePoint, + ServiceId: []byte(min.ServiceID), + Ttl: min.ExpiredAt - now.Unix(), + MinSafePoint: min.SafePoint, }, nil } // WatchGCSafePointV2 watch keyspaces gc safe point changes. -func (s *GrpcServer) WatchGCSafePointV2(request *pdpb.WatchGCSafePointV2Request, stream pdpb.PD_WatchGCSafePointV2Server) error { - ctx, cancel := context.WithCancel(s.Context()) - defer cancel() - revision := request.GetRevision() - // If the revision is compacted, will meet required revision has been compacted error. - // - If required revision < CompactRevision, we need to reload all configs to avoid losing data. - // - If required revision >= CompactRevision, just keep watching. - // Use WithPrevKV() to get the previous key-value pair when get Delete Event. - watchChan := s.client.Watch(ctx, keypath.GCSafePointV2Prefix(), clientv3.WithRev(revision), clientv3.WithPrefix()) - for { - select { - case <-ctx.Done(): - return nil - case res := <-watchChan: - if res.Err() != nil { - var resp pdpb.WatchGCSafePointV2Response - if revision < res.CompactRevision { - resp.Header = wrapErrorToHeader(pdpb.ErrorType_DATA_COMPACTED, - fmt.Sprintf("required watch revision: %d is smaller than current compact/min revision %d.", revision, res.CompactRevision)) - } else { - resp.Header = wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, - fmt.Sprintf("watch channel meet other error %s.", res.Err().Error())) - } - if err := stream.Send(&resp); err != nil { - return err - } - // Err() indicates that this WatchResponse holds a channel-closing error. - return res.Err() - } - revision = res.Header.GetRevision() - - safePointEvents := make([]*pdpb.SafePointEvent, 0, len(res.Events)) - for _, event := range res.Events { - gcSafePoint := &endpoint.GCSafePointV2{} - if err := json.Unmarshal(event.Kv.Value, gcSafePoint); err != nil { - return err - } - safePointEvents = append(safePointEvents, &pdpb.SafePointEvent{ - KeyspaceId: gcSafePoint.KeyspaceID, - SafePoint: gcSafePoint.SafePoint, - Type: pdpb.EventType(event.Type), - }) - } - if len(safePointEvents) > 0 { - if err := stream.Send(&pdpb.WatchGCSafePointV2Response{Header: wrapHeader(), Events: safePointEvents, Revision: res.Header.GetRevision()}); err != nil { - return err - } - } - } - } +// Deprecated: Poll GetAllKeyspacesGCStates instead. +// +//nolint:staticcheck +func (*GrpcServer) WatchGCSafePointV2(_ *pdpb.WatchGCSafePointV2Request, _ pdpb.PD_WatchGCSafePointV2Server) error { + log.Error("removed API WatchGCSafePointV2 is called") + return status.Errorf(codes.Unimplemented, "WatchGCSafePointV2 is obsolete. Poll GetAllKeyspacesGCStates instead if necessary") } // GetAllGCSafePointV2 return all gc safe point v2. -func (s *GrpcServer) GetAllGCSafePointV2(ctx context.Context, request *pdpb.GetAllGCSafePointV2Request) (*pdpb.GetAllGCSafePointV2Response, error) { +// Deprecated: Use GetAllKeyspacesGCStates instead. +// +//nolint:staticcheck +func (s *GrpcServer) GetAllGCSafePointV2(ctx context.Context, request *pdpb.GetAllGCSafePointV2Request) (resp *pdpb.GetAllGCSafePointV2Response, errRet error) { + log.Warn("deprecated API GetAllGCSafePointV2 is called") + defer func() { + if errRet != nil { + log.Error("deprecated API GetAllGCSafePointV2 encountered error", zap.Error(errRet)) + } else { + log.Warn("deprecated API GetAllGCSafePointV2 returned", zap.Stringers("resp-safe-point", resp.GetGcSafePoints()), zap.Int64("resp-revision", resp.GetRevision())) + } + }() + + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() + } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetAllGCSafePointV2(ctx, request) } @@ -208,57 +426,38 @@ func (s *GrpcServer) GetAllGCSafePointV2(ctx context.Context, request *pdpb.GetA return rsp.(*pdpb.GetAllGCSafePointV2Response), err } - startkey := keypath.GCSafePointV2Prefix() - endkey := clientv3.GetPrefixRangeEnd(startkey) - values, revision, err := s.loadRangeFromEtcd(startkey, endkey) - - gcSafePoints := make([]*pdpb.GCSafePointV2, 0, len(values)) - for _, value := range values { - jsonGcSafePoint := &endpoint.GCSafePointV2{} - if err = json.Unmarshal([]byte(value), jsonGcSafePoint); err != nil { - return nil, errs.ErrJSONUnmarshal.Wrap(err).GenWithStackByCause() - } - gcSafePoint := &pdpb.GCSafePointV2{ - KeyspaceId: jsonGcSafePoint.KeyspaceID, - GcSafePoint: jsonGcSafePoint.SafePoint, - } - log.Debug("get all gc safe point v2", - zap.Uint32("keyspace-id", jsonGcSafePoint.KeyspaceID), - zap.Uint64("gc-safe-point", jsonGcSafePoint.SafePoint)) - gcSafePoints = append(gcSafePoints, gcSafePoint) + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.GetAllGCSafePointV2Response{Header: notBootstrappedHeader()}, nil } + gcStates, err := s.gcStateManager.GetAllKeyspacesGCStates() if err != nil { return &pdpb.GetAllGCSafePointV2Response{ Header: wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), - }, err + }, nil + } + + gcSafePoints := make([]*pdpb.GCSafePointV2, 0, len(gcStates)) + for _, gcState := range gcStates { + if gcState.KeyspaceID == constant.NullKeyspaceID { + // The original GetAllGCSafePointV2 API doesn't return the null keyspace. To keep the behavior, we + // still exclude it. + continue + } + gcSafePoints = append(gcSafePoints, &pdpb.GCSafePointV2{ + KeyspaceId: gcState.KeyspaceID, + GcSafePoint: gcState.GCSafePoint, + }) } return &pdpb.GetAllGCSafePointV2Response{ Header: wrapHeader(), GcSafePoints: gcSafePoints, - Revision: revision, + Revision: 0, }, nil } -func (s *GrpcServer) loadRangeFromEtcd(startKey, endKey string) (values []string, revision int64, err error) { - var opOption []clientv3.OpOption - if endKey == "\x00" { - opOption = append(opOption, clientv3.WithPrefix()) - } else { - opOption = append(opOption, clientv3.WithRange(endKey)) - } - resp, err := etcdutil.EtcdKVGet(s.client, startKey, opOption...) - if err != nil { - return nil, 0, err - } - values = make([]string, 0, len(resp.Kvs)) - for _, item := range resp.Kvs { - values = append(values, string(item.Value)) - } - return values, resp.Header.Revision, nil -} - func getKeyspaceID(keyspaceScope *pdpb.KeyspaceScope) uint32 { if keyspaceScope == nil { return constant.NullKeyspaceID diff --git a/server/grpc_service.go b/server/grpc_service.go index 988e9f3be91..6fef05ee21f 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -47,7 +47,6 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/ratelimit" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/keypath" @@ -2155,40 +2154,6 @@ func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterReg }, nil } -// GetGCSafePoint implements gRPC PDServer. -func (s *GrpcServer) GetGCSafePoint(ctx context.Context, request *pdpb.GetGCSafePointRequest) (*pdpb.GetGCSafePointResponse, error) { - done, err := s.rateLimitCheck() - if err != nil { - return nil, err - } - if done != nil { - defer done() - } - fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { - return pdpb.NewPDClient(client).GetGCSafePoint(ctx, request) - } - if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { - return nil, err - } else if rsp != nil { - return rsp.(*pdpb.GetGCSafePointResponse), err - } - - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.GetGCSafePointResponse{Header: notBootstrappedHeader()}, nil - } - - safePoint, err := s.gcSafePointManager.LoadGCSafePoint() - if err != nil { - return nil, err - } - - return &pdpb.GetGCSafePointResponse{ - Header: wrapHeader(), - SafePoint: safePoint, - }, nil -} - // SyncRegions syncs the regions. func (s *GrpcServer) SyncRegions(stream pdpb.PD_SyncRegionsServer) error { if s.IsClosed() || s.cluster == nil { @@ -2208,103 +2173,6 @@ func (s *GrpcServer) SyncRegions(stream pdpb.PD_SyncRegionsServer) error { return s.cluster.GetRegionSyncer().Sync(ctx, stream) } -// UpdateGCSafePoint implements gRPC PDServer. -func (s *GrpcServer) UpdateGCSafePoint(ctx context.Context, request *pdpb.UpdateGCSafePointRequest) (*pdpb.UpdateGCSafePointResponse, error) { - done, err := s.rateLimitCheck() - if err != nil { - return nil, err - } - if done != nil { - defer done() - } - fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { - return pdpb.NewPDClient(client).UpdateGCSafePoint(ctx, request) - } - if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { - return nil, err - } else if rsp != nil { - return rsp.(*pdpb.UpdateGCSafePointResponse), err - } - - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.UpdateGCSafePointResponse{Header: notBootstrappedHeader()}, nil - } - - newSafePoint := request.GetSafePoint() - oldSafePoint, err := s.gcSafePointManager.UpdateGCSafePoint(newSafePoint) - if err != nil { - return nil, err - } - - if newSafePoint > oldSafePoint { - log.Info("updated gc safe point", - zap.Uint64("safe-point", newSafePoint)) - } else if newSafePoint < oldSafePoint { - log.Warn("trying to update gc safe point", - zap.Uint64("old-safe-point", oldSafePoint), - zap.Uint64("new-safe-point", newSafePoint)) - newSafePoint = oldSafePoint - } - - return &pdpb.UpdateGCSafePointResponse{ - Header: wrapHeader(), - NewSafePoint: newSafePoint, - }, nil -} - -// UpdateServiceGCSafePoint update the safepoint for specific service -func (s *GrpcServer) UpdateServiceGCSafePoint(ctx context.Context, request *pdpb.UpdateServiceGCSafePointRequest) (*pdpb.UpdateServiceGCSafePointResponse, error) { - done, err := s.rateLimitCheck() - if err != nil { - return nil, err - } - if done != nil { - defer done() - } - fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { - return pdpb.NewPDClient(client).UpdateServiceGCSafePoint(ctx, request) - } - if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { - return nil, err - } else if rsp != nil { - return rsp.(*pdpb.UpdateServiceGCSafePointResponse), err - } - - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.UpdateServiceGCSafePointResponse{Header: notBootstrappedHeader()}, nil - } - var storage endpoint.GCSafePointStorage = s.storage - if request.TTL <= 0 { - if err := storage.RemoveServiceGCSafePoint(string(request.ServiceId)); err != nil { - return nil, err - } - } - nowTSO, err := s.getGlobalTSO(ctx) - if err != nil { - return nil, err - } - now, _ := tsoutil.ParseTimestamp(nowTSO) - serviceID := string(request.ServiceId) - min, updated, err := s.gcSafePointManager.UpdateServiceGCSafePoint(serviceID, request.GetSafePoint(), request.GetTTL(), now) - if err != nil { - return nil, err - } - if updated { - log.Info("update service GC safe point", - zap.String("service-id", serviceID), - zap.Int64("expire-at", now.Unix()+request.GetTTL()), - zap.Uint64("safepoint", request.GetSafePoint())) - } - return &pdpb.UpdateServiceGCSafePointResponse{ - Header: wrapHeader(), - ServiceId: []byte(min.ServiceID), - TTL: min.ExpiredAt - now.Unix(), - MinSafePoint: min.SafePoint, - }, nil -} - // GetOperator gets information about the operator belonging to the specify region. func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorRequest) (*pdpb.GetOperatorResponse, error) { done, err := s.rateLimitCheck() diff --git a/server/server.go b/server/server.go index 24a150c7e05..206507447b9 100644 --- a/server/server.go +++ b/server/server.go @@ -166,14 +166,10 @@ type Server struct { encryptionKeyManager *encryption.Manager // for storage operation. storage storage.Storage - // safe point manager (to be deprecated) - gcSafePointManager *gc.SafePointManager // GC states manager gcStateManager *gc.GCStateManager // keyspace manager keyspaceManager *keyspace.Manager - // safe point V2 manager - safePointV2Manager *gc.SafePointV2Manager // keyspace group manager keyspaceGroupManager *keyspace.GroupManager // for basicCluster operation. @@ -473,7 +469,6 @@ func (s *Server) startServer(ctx context.Context) error { s.tsoProtoFactory = &tsoutil.TSOProtoFactory{} s.pdProtoFactory = &tsoutil.PDProtoFactory{} s.tsoAllocator = tso.NewAllocator(s.ctx, constant.DefaultKeyspaceGroupID, s.member, s.storage, s) - s.gcSafePointManager = gc.NewSafePointManager(s.storage, s.cfg.PDServerCfg) s.basicCluster = core.NewBasicCluster() s.cluster = cluster.NewRaftCluster(ctx, s.GetMember(), s.GetBasicCluster(), s.GetStorage(), syncer.NewRegionSyncer(s), s.client, s.httpClient, s.tsoAllocator) keyspaceIDAllocator := id.NewAllocator(&id.AllocatorParams{ @@ -487,7 +482,6 @@ func (s *Server) startServer(ctx context.Context) error { } s.keyspaceManager = keyspace.NewKeyspaceManager(s.ctx, s.storage, s.cluster, keyspaceIDAllocator, &s.cfg.Keyspace, s.keyspaceGroupManager) s.gcStateManager = gc.NewGCStateManager(s.storage.GetGCStateProvider(), s.cfg.PDServerCfg, s.keyspaceManager) - s.safePointV2Manager = gc.NewSafePointManagerV2(s.ctx, s.storage, s.storage, s.storage) s.hbStreams = hbstream.NewHeartbeatStreams(ctx, "", s.cluster) // initial hot_region_storage in here. @@ -910,11 +904,6 @@ func (s *Server) SetKeyspaceManager(keyspaceManager *keyspace.Manager) { s.keyspaceManager = keyspaceManager } -// GetSafePointV2Manager returns the safe point v2 manager of server. -func (s *Server) GetSafePointV2Manager() *gc.SafePointV2Manager { - return s.safePointV2Manager -} - // GetKeyspaceGroupManager returns the keyspace group manager of server. func (s *Server) GetKeyspaceGroupManager() *keyspace.GroupManager { return s.keyspaceGroupManager diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 14172378a24..32bbadb5798 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -2042,24 +2042,74 @@ func (s *clientStatefulTestSuite) checkGCBarrier(re *require.Assertions, keyspac } } -func (s *clientStatefulTestSuite) TestUpdateGCSafePoint() { +func (s *clientStatefulTestSuite) testUpdateGCSafePointImpl(keyspaceID uint32) { re := s.Require() - s.checkGCSafePoint(re, constants.NullKeyspaceID, 0) - for _, safePoint := range []uint64{0, 1, 2, 3, 233, 23333, 233333333333, math.MaxUint64} { - newSafePoint, err := s.client.UpdateGCSafePoint(context.Background(), safePoint) + client := s.client + if keyspaceID != constants.NullKeyspaceID { + // UpdateGCSafePoint works according to the keyspace property set in the client. + client = utils.SetupClientWithKeyspaceID(context.Background(), re, keyspaceID, s.srv.GetEndpoints()) + defer client.Close() + } + + s.checkGCSafePoint(re, keyspaceID, 0) + for _, gcSafePoint := range []uint64{0, 1, 2, 3, 233, 23333, 233333333333, math.MaxUint64} { + // Now GC safe point is not allowed to be advanced before advancing the txn safe point. Advance txn safe point + // first. + _, err := s.client.GetGCInternalController(keyspaceID).AdvanceTxnSafePoint(context.Background(), gcSafePoint) re.NoError(err) - re.Equal(safePoint, newSafePoint) - s.checkGCSafePoint(re, constants.NullKeyspaceID, safePoint) + newSafePoint, err := client.UpdateGCSafePoint(context.Background(), gcSafePoint) //nolint:staticcheck + re.NoError(err) + re.Equal(gcSafePoint, newSafePoint) + s.checkGCSafePoint(re, keyspaceID, gcSafePoint) } // If the new safe point is less than the old one, it should not be updated. - newSafePoint, err := s.client.UpdateGCSafePoint(context.Background(), 1) + newSafePoint, err := client.UpdateGCSafePoint(context.Background(), 1) //nolint:staticcheck re.Equal(uint64(math.MaxUint64), newSafePoint) re.NoError(err) - s.checkGCSafePoint(re, constants.NullKeyspaceID, math.MaxUint64) + s.checkGCSafePoint(re, keyspaceID, math.MaxUint64) } -func (s *clientStatefulTestSuite) TestUpdateServiceGCSafePoint() { +func (s *clientStatefulTestSuite) TestUpdateGCSafePoint() { + s.prepareKeyspacesForGCTest() + for _, keyspaceID := range []uint32{constants.NullKeyspaceID, 1, 2} { + s.testUpdateGCSafePointImpl(keyspaceID) + } +} + +func (s *clientStatefulTestSuite) testUpdateServiceGCSafePointImpl(keyspaceID uint32) { re := s.Require() + + client := s.client + if keyspaceID != constants.NullKeyspaceID { + // UpdateServiceGCSafePoint works according to the keyspace property set in the client. + client = utils.SetupClientWithKeyspaceID(context.Background(), re, keyspaceID, s.srv.GetEndpoints()) + defer client.Close() + } + + loadMinServiceGCSafePoint := func() *endpoint.ServiceSafePoint { + res, _, err := s.srv.GetGCStateManager().CompatibleUpdateServiceGCSafePoint(keyspaceID, "_", 0, 0, time.Now()) + re.NoError(err) + return res + } + + // Suppress the unuseful lint warning. + //nolint:unparam + loadServiceGCSafePointByServiceID := func(serviceID string) *endpoint.ServiceSafePoint { + gcStates, err := s.srv.GetGCStateManager().GetGCState(keyspaceID) + re.NoError(err) + for _, b := range gcStates.GCBarriers { + if b.BarrierID == serviceID { + return b.ToServiceSafePoint(keyspaceID) + } + } + return nil + } + + // Wrap in a function to avoid writing the nolint directive everywhere. + updateServiceGCSafePoint := func(serviceID string, ttl int64, safePoint uint64) (uint64, error) { + return client.UpdateServiceGCSafePoint(context.Background(), serviceID, ttl, safePoint) //nolint:staticcheck + } + serviceSafePoints := []struct { ServiceID string TTL int64 @@ -2070,113 +2120,107 @@ func (s *clientStatefulTestSuite) TestUpdateServiceGCSafePoint() { {"c", 1000, 3}, } for _, ssp := range serviceSafePoints { - min, err := s.client.UpdateServiceGCSafePoint(context.Background(), - ssp.ServiceID, 1000, ssp.SafePoint) + min, err := updateServiceGCSafePoint(ssp.ServiceID, 1000, ssp.SafePoint) re.NoError(err) // An service safepoint of ID "gc_worker" is automatically initialized as 0 re.Equal(uint64(0), min) } - min, err := s.client.UpdateServiceGCSafePoint(context.Background(), - "gc_worker", math.MaxInt64, 10) + min, err := updateServiceGCSafePoint("gc_worker", math.MaxInt64, 10) re.NoError(err) re.Equal(uint64(1), min) - min, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "a", 1000, 4) + // Note that as the service safe points became a compatibility layer over the GC barriers and the txn safe point, + // the (simulated) service safe point of "gc_worker" is no longer able to be advanced over the minimal existing + // GC barrier. + + min, err = updateServiceGCSafePoint("a", 1000, 4) + re.NoError(err) + re.Equal(uint64(1), min) + min, err = updateServiceGCSafePoint("gc_worker", math.MaxInt64, 10) re.NoError(err) re.Equal(uint64(2), min) - min, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "b", -100, 2) + min, err = updateServiceGCSafePoint("b", -100, 2) + re.NoError(err) + re.Equal(uint64(2), min) + min, err = updateServiceGCSafePoint("gc_worker", math.MaxInt64, 10) re.NoError(err) re.Equal(uint64(3), min) // Minimum safepoint does not regress - min, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "b", 1000, 2) + min, err = updateServiceGCSafePoint("b", 1000, 2) re.NoError(err) re.Equal(uint64(3), min) - // Update only the TTL of the minimum safepoint - oldMinSsp, err := s.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - re.NoError(err) + // Update only the TTL of the service safe point "c" + oldMinSsp := loadServiceGCSafePointByServiceID("c") re.Equal("c", oldMinSsp.ServiceID) re.Equal(uint64(3), oldMinSsp.SafePoint) - min, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "c", 2000, 3) + min, err = updateServiceGCSafePoint("c", 2000, 3) re.NoError(err) re.Equal(uint64(3), min) - minSsp, err := s.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - re.NoError(err) + minSsp := loadServiceGCSafePointByServiceID("c") re.Equal("c", minSsp.ServiceID) - re.Equal(uint64(3), oldMinSsp.SafePoint) + re.Equal(uint64(3), minSsp.SafePoint) s.GreaterOrEqual(minSsp.ExpiredAt-oldMinSsp.ExpiredAt, int64(1000)) // Shrinking TTL is also allowed - min, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "c", 1, 3) + min, err = updateServiceGCSafePoint("c", 1, 3) re.NoError(err) re.Equal(uint64(3), min) - minSsp, err = s.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) + minSsp = loadServiceGCSafePointByServiceID("c") re.NoError(err) re.Equal("c", minSsp.ServiceID) re.Less(minSsp.ExpiredAt, oldMinSsp.ExpiredAt) // TTL can be infinite (represented by math.MaxInt64) - min, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "c", math.MaxInt64, 3) + min, err = updateServiceGCSafePoint("c", math.MaxInt64, 3) re.NoError(err) re.Equal(uint64(3), min) - minSsp, err = s.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) + minSsp = loadServiceGCSafePointByServiceID("c") re.NoError(err) re.Equal("c", minSsp.ServiceID) re.Equal(minSsp.ExpiredAt, int64(math.MaxInt64)) // Delete "a" and "c" - min, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "c", -1, 3) + _, err = updateServiceGCSafePoint("c", -1, 3) re.NoError(err) - re.Equal(uint64(4), min) - min, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "a", -1, 4) + _, err = updateServiceGCSafePoint("a", -1, 4) + re.NoError(err) + // Now the service safe point of gc_worker can be advanced as other service safe points are all deleted. + min, err = updateServiceGCSafePoint("gc_worker", math.MaxInt64, 10) re.NoError(err) - // Now gc_worker is the only remaining service safe point. re.Equal(uint64(10), min) // gc_worker cannot be deleted. - _, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "gc_worker", -1, 10) + _, err = updateServiceGCSafePoint("gc_worker", -1, 10) re.Error(err) // Cannot set non-infinity TTL for gc_worker - _, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "gc_worker", 10000000, 10) + _, err = updateServiceGCSafePoint("gc_worker", 10000000, 10) re.Error(err) // Service safepoint must have a non-empty ID - _, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "", 1000, 15) + _, err = updateServiceGCSafePoint("", 1000, 15) re.Error(err) // Put some other safepoints to test fixing gc_worker's safepoint when there exists other safepoints. - _, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "a", 1000, 11) + _, err = updateServiceGCSafePoint("a", 1000, 11) re.NoError(err) - _, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "b", 1000, 12) + _, err = updateServiceGCSafePoint("b", 1000, 12) re.NoError(err) - _, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "c", 1000, 13) + _, err = updateServiceGCSafePoint("c", 1000, 13) re.NoError(err) // Force set invalid ttl to gc_worker gcWorkerKey := keypath.ServiceGCSafePointPath("gc_worker") { gcWorkerSsp := &endpoint.ServiceSafePoint{ - ServiceID: "gc_worker", - ExpiredAt: -12345, - SafePoint: 10, + ServiceID: "gc_worker", + ExpiredAt: -12345, + SafePoint: 10, + KeyspaceID: keyspaceID, } value, err := json.Marshal(gcWorkerSsp) re.NoError(err) @@ -2184,39 +2228,26 @@ func (s *clientStatefulTestSuite) TestUpdateServiceGCSafePoint() { re.NoError(err) } - minSsp, err = s.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) + minSsp = loadMinServiceGCSafePoint() re.NoError(err) re.Equal("gc_worker", minSsp.ServiceID) re.Equal(uint64(10), minSsp.SafePoint) re.Equal(int64(math.MaxInt64), minSsp.ExpiredAt) - // Force delete gc_worker, then the min service safe point was 11 of "a" in previous implementation, but 0 - // for compatibility of txn safe point. - err = s.srv.GetStorage().Remove(gcWorkerKey) - re.NoError(err) - minSsp, err = s.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - re.NoError(err) - re.Equal(uint64(0), minSsp.SafePoint) - // Advancing txn safe point also affects the gc_worker's service safe point. - _, err = s.client.GetGCInternalController(constants.NullKeyspaceID).AdvanceTxnSafePoint(context.Background(), 11) + _, err = client.GetGCInternalController(keyspaceID).AdvanceTxnSafePoint(context.Background(), 11) re.NoError(err) - minSsp, err = s.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) + minSsp = loadMinServiceGCSafePoint() re.NoError(err) re.Equal(uint64(11), minSsp.SafePoint) +} - // After calling LoadMinServiceGCS when "gc_worker"'s service safepoint is missing, "gc_worker"'s service safepoint - // will be newly created. - // Increase "a" so that "gc_worker" is the only minimum that will be returned by LoadMinServiceGCSafePoint. - _, err = s.client.UpdateServiceGCSafePoint(context.Background(), - "a", 1000, 14) - re.NoError(err) +func (s *clientStatefulTestSuite) TestUpdateServiceGCSafePoint() { + s.prepareKeyspacesForGCTest() - minSsp, err = s.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - re.NoError(err) - re.Equal("gc_worker", minSsp.ServiceID) - re.Equal(uint64(11), minSsp.SafePoint) - re.Equal(int64(math.MaxInt64), minSsp.ExpiredAt) + for _, keyspaceID := range []uint32{constants.NullKeyspaceID, 1, 2} { + s.testUpdateServiceGCSafePointImpl(keyspaceID) + } } func (s *clientStatefulTestSuite) prepareKeyspacesForGCTest() { diff --git a/tests/integrations/client/gc_api_v2_client_test.go b/tests/integrations/client/gc_api_v2_client_test.go deleted file mode 100644 index 3c706f84068..00000000000 --- a/tests/integrations/client/gc_api_v2_client_test.go +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2023 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package client_test - -import ( - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - clientv3 "go.etcd.io/etcd/client/v3" - "go.uber.org/zap" - "google.golang.org/grpc" - - "github.com/pingcap/failpoint" - "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/pingcap/log" - - pd "github.com/tikv/pd/client" - "github.com/tikv/pd/client/pkg/caller" - "github.com/tikv/pd/pkg/utils/assertutil" - "github.com/tikv/pd/pkg/utils/keypath" - "github.com/tikv/pd/pkg/utils/testutil" - "github.com/tikv/pd/server" - "github.com/tikv/pd/tests" -) - -// gcClientTestReceiver is the pdpb.PD_WatchGCSafePointV2Server mock for testing. -type gcClientTestReceiver struct { - re *require.Assertions - grpc.ServerStream -} - -// Send is the mock implementation for pdpb.PD_WatchGCSafePointV2Server's Send. -// Instead of sending the response to the client, it will check the response. -// In testing, we will set all keyspace's safe point to be equal to its id, -// and this mock verifies that the response is correct. -func (s gcClientTestReceiver) Send(m *pdpb.WatchGCSafePointV2Response) error { - log.Info("received", zap.Any("received", m.GetEvents())) - for _, change := range m.GetEvents() { - s.re.Equal(change.SafePoint, uint64(change.KeyspaceId)) - } - return nil -} - -type gcClientTestSuite struct { - suite.Suite - server *server.GrpcServer - client pd.Client - cleanup testutil.CleanupFunc - gcSafePointV2Prefix string -} - -func TestGcClientTestSuite(t *testing.T) { - suite.Run(t, new(gcClientTestSuite)) -} - -func (suite *gcClientTestSuite) SetupSuite() { - re := suite.Require() - var ( - err error - gsi *server.Server - ) - checker := assertutil.NewChecker() - checker.FailNow = func() {} - gsi, suite.cleanup, err = tests.NewServer(re, checker) - suite.server = &server.GrpcServer{Server: gsi} - re.NoError(err) - addr := suite.server.GetAddr() - suite.client, err = pd.NewClientWithContext(suite.server.Context(), - caller.TestComponent, - []string{addr}, pd.SecurityOption{}, - ) - re.NoError(err) - suite.gcSafePointV2Prefix = keypath.GCSafePointV2Prefix() - // Enable the fail-point to skip checking keyspace validity. - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/gc/checkKeyspace", "return(true)")) -} - -func (suite *gcClientTestSuite) TearDownSuite() { - re := suite.Require() - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/gc/checkKeyspace")) - suite.cleanup() - suite.client.Close() -} - -func (suite *gcClientTestSuite) TearDownTest() { - suite.CleanupEtcdGCPath() -} - -func (suite *gcClientTestSuite) CleanupEtcdGCPath() { - re := suite.Require() - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.gcSafePointV2Prefix, clientv3.WithPrefix()) - re.NoError(err) -} - -func (suite *gcClientTestSuite) TestWatch1() { - re := suite.Require() - receiver := gcClientTestReceiver{re: suite.Require()} - go func() { - // TODO: fix this test - // It will return "unexpected end of JSON input" because watch value is empty and exit watch loop directly. - // And we also need to use wait group to manage this goroutine. - _ = suite.server.WatchGCSafePointV2(&pdpb.WatchGCSafePointV2Request{ - Revision: 0, - }, receiver) - }() - - // Init gc safe points as index value of keyspace 0 ~ 5. - for i := range 6 { - suite.mustUpdateSafePoint(re, uint32(i), uint64(i)) - } - - // delete gc safe points of keyspace 3 ~ 5. - for i := 3; i < 6; i++ { - suite.mustDeleteSafePoint(re, uint32(i)) - } - - // check gc safe point equal to keyspace id for keyspace 0 ~ 2 . - for i := range 3 { - re.Equal(uint64(i), suite.mustLoadSafePoint(re, uint32(i))) - } - - // check gc safe point is 0 for keyspace 3 ~ 5 after delete. - for i := 3; i < 6; i++ { - re.Equal(uint64(0), suite.mustLoadSafePoint(re, uint32(i))) - } -} - -func (suite *gcClientTestSuite) TestClientWatchWithRevision() { - suite.testClientWatchWithRevision(false) - suite.testClientWatchWithRevision(true) -} - -// nolint:revive -func (suite *gcClientTestSuite) testClientWatchWithRevision(fromNewRevision bool) { - re := suite.Require() - testKeyspaceID := uint32(100) - initGCSafePoint := uint64(50) - updatedGCSafePoint := uint64(100) - - // Init gc safe point. - suite.mustUpdateSafePoint(re, testKeyspaceID, initGCSafePoint) - - // Get the initial revision. - initRevision := suite.mustGetRevision(re, testKeyspaceID) - - // Update the gc safe point. - suite.mustUpdateSafePoint(re, testKeyspaceID, updatedGCSafePoint) - - // Get the revision of the updated gc safe point. - updatedRevision := suite.mustGetRevision(re, testKeyspaceID) - - // Set the start revision of the watch request based on fromNewRevision. - startRevision := initRevision - if fromNewRevision { - startRevision = updatedRevision - } - watchChan, err := suite.client.WatchGCSafePointV2(suite.server.Context(), startRevision) //nolint:staticcheck - re.NoError(err) - - timer := time.NewTimer(time.Second) - defer timer.Stop() - isFirstUpdate := true - runTest := false - for { - select { - case <-timer.C: - re.True(runTest) - return - case res := <-watchChan: - for _, r := range res { - re.Equal(r.GetKeyspaceId(), testKeyspaceID) - if fromNewRevision { - // If fromNewRevision, first response should be the updated gc safe point. - re.Equal(r.GetSafePoint(), updatedGCSafePoint) - } else if isFirstUpdate { - isFirstUpdate = false - re.Equal(r.GetSafePoint(), initGCSafePoint) - } else { - re.Equal(r.GetSafePoint(), updatedGCSafePoint) - continue - } - } - runTest = true - } - } -} - -// mustUpdateSafePoint updates the gc safe point of the given keyspace id. -func (suite *gcClientTestSuite) mustUpdateSafePoint(re *require.Assertions, keyspaceID uint32, safePoint uint64) { - client, err := pd.NewClientWithKeyspace(suite.server.Context(), - caller.TestComponent, - keyspaceID, - []string{suite.server.GetAddr()}, - pd.SecurityOption{}, - ) - re.NoError(err) - _, err = client.UpdateGCSafePoint(suite.server.Context(), safePoint) - re.NoError(err) -} - -// mustLoadSafePoint loads the gc safe point of the given keyspace id. -func (suite *gcClientTestSuite) mustLoadSafePoint(re *require.Assertions, keyspaceID uint32) uint64 { - res, err := suite.server.GetSafePointV2Manager().LoadGCSafePoint(keyspaceID) - re.NoError(err) - return res.SafePoint -} - -// mustDeleteSafePoint deletes the gc safe point of the given keyspace id. -func (suite *gcClientTestSuite) mustDeleteSafePoint(re *require.Assertions, keyspaceID uint32) { - safePointPath := keypath.GCSafePointV2Path(keyspaceID) - _, err := suite.server.GetClient().Delete(suite.server.Context(), safePointPath) - re.NoError(err) -} - -// mustGetRevision gets the revision of the given keyspace's gc safe point. -func (suite *gcClientTestSuite) mustGetRevision(re *require.Assertions, keyspaceID uint32) int64 { - safePointPath := keypath.GCSafePointV2Path(keyspaceID) - res, err := suite.server.GetClient().Get(suite.server.Context(), safePointPath) - re.NoError(err) - return res.Header.GetRevision() -} diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 8d16191d6a1..6a32c652723 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -38,6 +38,7 @@ import ( "github.com/tikv/pd/client/pkg/retry" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/keyspace/constant" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" @@ -993,21 +994,22 @@ func (suite *httpClientTestSuite) TestGetGCSafePoint() { defer cancel() // adding some safepoints to the server + now := time.Now().Truncate(time.Second) list := &api.ListServiceGCSafepoint{ ServiceGCSafepoints: []*endpoint.ServiceSafePoint{ { ServiceID: "AAA", - ExpiredAt: time.Now().Unix() + 10, + ExpiredAt: now.Unix() + 10, SafePoint: 1, }, { ServiceID: "BBB", - ExpiredAt: time.Now().Unix() + 10, + ExpiredAt: now.Unix() + 10, SafePoint: 2, }, { ServiceID: "CCC", - ExpiredAt: time.Now().Unix() + 10, + ExpiredAt: now.Unix() + 10, SafePoint: 3, }, }, @@ -1015,12 +1017,14 @@ func (suite *httpClientTestSuite) TestGetGCSafePoint() { MinServiceGcSafepoint: 1, } - storage := suite.cluster.GetLeaderServer().GetServer().GetStorage() + gcStateManager := suite.cluster.GetLeaderServer().GetServer().GetGCStateManager() for _, ssp := range list.ServiceGCSafepoints { - err := storage.SaveServiceGCSafePoint(ssp) + _, _, err := gcStateManager.CompatibleUpdateServiceGCSafePoint(constant.NullKeyspaceID, ssp.ServiceID, ssp.SafePoint, ssp.ExpiredAt-now.Unix(), now) re.NoError(err) } - err := storage.SaveGCSafePoint(1) + _, err := gcStateManager.AdvanceTxnSafePoint(constant.NullKeyspaceID, 1, now) + re.NoError(err) + _, _, err = gcStateManager.AdvanceGCSafePoint(constant.NullKeyspaceID, 1) re.NoError(err) // get the safepoints and start testing @@ -1056,9 +1060,11 @@ func (suite *httpClientTestSuite) TestGetGCSafePoint() { re.Equal(uint64(0), l.MinServiceGcSafepoint) re.Empty(l.ServiceGCSafepoints) - // try delete gc_worker, should get an error + // Deleting "gc_worker" should result in an error in earlier version. As the service safe point becomes a + // compatibility layer over GC barriers, it won't take any effect except that possibly deleting the residual + // service safe point of "gc_worker" that was written by previous version. _, err = client.DeleteGCSafePoint(ctx, "gc_worker") - re.Error(err) + re.NoError(err) // try delete some non-exist safepoints, should return normally var msg string diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index dc3e630cd6d..b3cf8e092e7 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -388,7 +388,7 @@ func TestForwardTSOUnexpectedToFollower1(t *testing.T) { suite.checkForwardTSOUnexpectedToFollower(func() { // unary call will retry internally // try to update gc safe point - min, err := suite.pdClient.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 1) + min, err := suite.pdClient.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 1) //nolint:staticcheck re.NoError(err) re.Equal(uint64(0), min) }) @@ -476,7 +476,7 @@ func (suite *PDServiceForward) checkUnavailableTSO(re *require.Assertions) { _, _, err := suite.pdClient.GetTS(suite.ctx) re.Error(err) // try to update gc safe point - _, err = suite.pdClient.UpdateServiceGCSafePoint(suite.ctx, "a", 1000, 1) + _, err = suite.pdClient.UpdateServiceGCSafePoint(suite.ctx, "a", 1000, 1) //nolint:staticcheck re.Error(err) // try to set external ts err = suite.pdClient.SetExternalTimestamp(suite.ctx, 1000) @@ -489,7 +489,7 @@ func (suite *PDServiceForward) checkAvailableTSO(re *require.Assertions) { _, _, err := suite.pdClient.GetTS(suite.ctx) re.NoError(err) // try to update gc safe point - min, err := suite.pdClient.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 1) + min, err := suite.pdClient.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 1) //nolint:staticcheck re.NoError(err) re.Equal(uint64(0), min) // try to set external ts @@ -524,7 +524,7 @@ func TestForwardTsoConcurrently(t *testing.T) { defer pdClient.Close() for range 10 { testutil.Eventually(re, func() bool { - min, err := pdClient.UpdateServiceGCSafePoint(context.Background(), fmt.Sprintf("service-%d", i), 1000, 1) + min, err := pdClient.UpdateServiceGCSafePoint(context.Background(), fmt.Sprintf("service-%d", i), 1000, 1) //nolint:staticcheck return err == nil && min == 0 }) } @@ -568,7 +568,7 @@ func BenchmarkForwardTsoConcurrently(b *testing.B) { go func() { defer wg.Done() for range 1000 { - min, err := client.UpdateServiceGCSafePoint(context.Background(), fmt.Sprintf("service-%d", i), 1000, 1) + min, err := client.UpdateServiceGCSafePoint(context.Background(), fmt.Sprintf("service-%d", i), 1000, 1) //nolint:staticcheck re.NoError(err) re.Equal(uint64(0), min) } diff --git a/tests/integrations/realcluster/etcd_key_test.go b/tests/integrations/realcluster/etcd_key_test.go index 5762dc58bdd..5eab3941940 100644 --- a/tests/integrations/realcluster/etcd_key_test.go +++ b/tests/integrations/realcluster/etcd_key_test.go @@ -109,7 +109,7 @@ func (s *etcdKeySuite) TestEtcdKey() { } t := s.T() // call `UpdateGCSafePoint` to create the key `/pd//gc/safe_point`. - _, err := s.cli.UpdateGCSafePoint(context.Background(), 1) + _, err := s.cli.UpdateGCSafePoint(context.Background(), 1) //nolint:staticcheck re.NoError(err) endpoints := getPDEndpoints(re) diff --git a/tests/server/api/service_gc_safepoint_test.go b/tests/server/api/service_gc_safepoint_test.go index 35d5af951fd..34d43ce1885 100644 --- a/tests/server/api/service_gc_safepoint_test.go +++ b/tests/server/api/service_gc_safepoint_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" + "github.com/tikv/pd/pkg/keyspace/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/testutil" @@ -63,33 +64,39 @@ func (suite *serviceGCSafepointTestSuite) checkServiceGCSafepoint(cluster *tests leader := cluster.GetLeaderServer() sspURL := leader.GetAddr() + "/pd/api/v1/gc/safepoint" - storage := leader.GetServer().GetStorage() + gcStateManager := leader.GetServer().GetGCStateManager() + now := time.Now().Truncate(time.Second) list := &api.ListServiceGCSafepoint{ ServiceGCSafepoints: []*endpoint.ServiceSafePoint{ { - ServiceID: "a", - ExpiredAt: time.Now().Unix() + 10, - SafePoint: 1, + ServiceID: "a", + ExpiredAt: now.Unix() + 10, + SafePoint: 1, + KeyspaceID: constant.NullKeyspaceID, }, { - ServiceID: "b", - ExpiredAt: time.Now().Unix() + 10, - SafePoint: 2, + ServiceID: "b", + ExpiredAt: now.Unix() + 10, + SafePoint: 2, + KeyspaceID: constant.NullKeyspaceID, }, { - ServiceID: "c", - ExpiredAt: time.Now().Unix() + 10, - SafePoint: 3, + ServiceID: "c", + ExpiredAt: now.Unix() + 10, + SafePoint: 3, + KeyspaceID: constant.NullKeyspaceID, }, }, GCSafePoint: 1, MinServiceGcSafepoint: 1, } for _, ssp := range list.ServiceGCSafepoints { - err := storage.SaveServiceGCSafePoint(ssp) + _, _, err := gcStateManager.CompatibleUpdateServiceGCSafePoint(constant.NullKeyspaceID, ssp.ServiceID, ssp.SafePoint, 10, now) re.NoError(err) } - err := storage.SaveGCSafePoint(1) + _, err := gcStateManager.AdvanceTxnSafePoint(constant.NullKeyspaceID, 1, now) + re.NoError(err) + _, _, err = gcStateManager.AdvanceGCSafePoint(constant.NullKeyspaceID, 1) re.NoError(err) res, err := tests.TestDialClient.Get(sspURL) @@ -103,7 +110,12 @@ func (suite *serviceGCSafepointTestSuite) checkServiceGCSafepoint(cluster *tests err = testutil.CheckDelete(tests.TestDialClient, sspURL+"/a", testutil.StatusOK(re)) re.NoError(err) - left, err := storage.LoadAllServiceGCSafePoints() + state, err := gcStateManager.GetGCState(constant.NullKeyspaceID) re.NoError(err) - re.Equal(list.ServiceGCSafepoints[1:], left) + left := state.GCBarriers + leftSsps := make([]*endpoint.ServiceSafePoint, 0, len(left)) + for _, barrier := range left { + leftSsps = append(leftSsps, barrier.ToServiceSafePoint(constant.NullKeyspaceID)) + } + re.Equal(list.ServiceGCSafepoints[1:], leftSsps) } diff --git a/tools/pd-api-bench/cases/cases.go b/tools/pd-api-bench/cases/cases.go index 5d63b2356f6..148a12976d4 100644 --- a/tools/pd-api-bench/cases/cases.go +++ b/tools/pd-api-bench/cases/cases.go @@ -253,6 +253,7 @@ func newUpdateGCSafePoint() func() GRPCCase { func (*updateGCSafePoint) unary(ctx context.Context, cli pd.Client) error { s := time.Now().Unix() + //nolint:staticcheck _, err := cli.UpdateGCSafePoint(ctx, uint64(s)) if err != nil { return err @@ -278,6 +279,7 @@ func newUpdateServiceGCSafePoint() func() GRPCCase { func (*updateServiceGCSafePoint) unary(ctx context.Context, cli pd.Client) error { s := time.Now().Unix() id := rand.Int63n(100) + 1 + //nolint:staticcheck _, err := cli.UpdateServiceGCSafePoint(ctx, strconv.FormatInt(id, 10), id, uint64(s)) if err != nil { return err diff --git a/tools/pd-ctl/tests/safepoint/safepoint_test.go b/tools/pd-ctl/tests/safepoint/safepoint_test.go index 5a7ba8a2bc8..887f62fe846 100644 --- a/tools/pd-ctl/tests/safepoint/safepoint_test.go +++ b/tools/pd-ctl/tests/safepoint/safepoint_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/keyspace/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/server/api" pdTests "github.com/tikv/pd/tests" @@ -46,21 +47,22 @@ func TestSafepoint(t *testing.T) { cmd := ctl.GetRootCmd() // add some gc_safepoint to the server + now := time.Now().Truncate(time.Second) list := &api.ListServiceGCSafepoint{ ServiceGCSafepoints: []*endpoint.ServiceSafePoint{ { ServiceID: "AAA", - ExpiredAt: time.Now().Unix() + 10, + ExpiredAt: now.Unix() + 10, SafePoint: 1, }, { ServiceID: "BBB", - ExpiredAt: time.Now().Unix() + 10, + ExpiredAt: now.Unix() + 10, SafePoint: 2, }, { ServiceID: "CCC", - ExpiredAt: time.Now().Unix() + 10, + ExpiredAt: now.Unix() + 10, SafePoint: 3, }, }, @@ -68,12 +70,15 @@ func TestSafepoint(t *testing.T) { MinServiceGcSafepoint: 1, } - storage := leaderServer.GetServer().GetStorage() + gcStateManager := leaderServer.GetServer().GetGCStateManager() for _, ssp := range list.ServiceGCSafepoints { - err := storage.SaveServiceGCSafePoint(ssp) + _, _, err = gcStateManager.CompatibleUpdateServiceGCSafePoint(constant.NullKeyspaceID, ssp.ServiceID, ssp.SafePoint, ssp.ExpiredAt-now.Unix(), now) re.NoError(err) } - storage.SaveGCSafePoint(1) + _, err = gcStateManager.AdvanceTxnSafePoint(constant.NullKeyspaceID, 1, now) + re.NoError(err) + _, _, err = gcStateManager.AdvanceGCSafePoint(constant.NullKeyspaceID, 1) + re.NoError(err) // get the safepoints args := []string{"-u", pdAddr, "service-gc-safepoint"} @@ -121,19 +126,21 @@ func TestSafepoint(t *testing.T) { re.Equal(uint64(0), ll.MinServiceGcSafepoint) re.Empty(ll.ServiceGCSafepoints) - // try delete the "gc_worker", should get an error message + // try delete the "gc_worker" args = []string{"-u", pdAddr, "service-gc-safepoint", "delete", "gc_worker"} output, err = tests.ExecuteCommand(cmd, args...) re.NoError(err) - // output should be an error message - re.Equal("Failed to delete service GC safepoint: request pd http api failed with status: '500 Internal Server Error', body: '\"cannot remove service safe point of gc_worker\"'\n", string(output)) + // This should be rejected previously, but as GC barrier became the replacement of services safe points and we no + // longer require the service safe point of "gc_worker", the deletion is now still allowed. + var msg string + re.NoError(json.Unmarshal(output, &msg)) + re.Equal("Delete service GC safepoint successfully.", msg) // try delete a non-exist safepoint, should return normally args = []string{"-u", pdAddr, "service-gc-safepoint", "delete", "non_exist"} output, err = tests.ExecuteCommand(cmd, args...) re.NoError(err) - var msg string re.NoError(json.Unmarshal(output, &msg)) re.Equal("Delete service GC safepoint successfully.", msg) }