statistics: delete cluster status metrics on store deletion (#9945)

close #9942

Signed-off-by: okjiang <819421878@qq.com>

Author: okJiang

pkg/core/store.go

@@ -244,6 +244,14 @@ func (s *StoreInfo) IsTiFlashCompute() bool {
 	return IsStoreContainLabel(s.GetMeta(), EngineKey, EngineTiFlashCompute)
 }
 
+// Engine returns the engine type of the store.
+func (s *StoreInfo) Engine() string {
+	if s.IsTiKV() {
+		return EngineTiKV
+	}
+	return EngineTiFlash
+}
+
 // IsUp returns true if store is serving or preparing.
 func (s *StoreInfo) IsUp() bool {
 	return s.IsServing() || s.IsPreparing()

pkg/mcs/router/server/meta/watcher.go

@@ -95,6 +95,7 @@ func (w *Watcher) initializeStoreWatcher() error {
 		}
 		origin := w.basicCluster.GetStore(storeID)
 		if origin != nil {
+			statistics.DeleteClusterStatusMetrics(origin)
 			w.basicCluster.DeleteStore(origin)
 			log.Info("delete store meta", zap.Uint64("store-id", storeID))
 		}

pkg/mcs/scheduling/server/meta/watcher.go

@@ -95,6 +95,7 @@ func (w *Watcher) initializeStoreWatcher() error {
 		}
 		origin := w.basicCluster.GetStore(storeID)
 		if origin != nil {
+			statistics.DeleteClusterStatusMetrics(origin)
 			w.basicCluster.DeleteStore(origin)
 			log.Info("delete store meta", zap.Uint64("store-id", storeID))
 		}

pkg/statistics/metrics.go

@@ -14,7 +14,13 @@
 
 package statistics
 
-import "github.com/prometheus/client_golang/prometheus"
+import (
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/tikv/pd/pkg/core"
+)
 
 var (
 	hotCacheStatusGauge = prometheus.NewGaugeVec(
@@ -118,3 +124,11 @@ func init() {
 	prometheus.MustRegister(regionAbnormalPeerDuration)
 	prometheus.MustRegister(hotPeerSummary)
 }
+
+// DeleteClusterStatusMetrics deletes the cluster status metrics of a store.
+func DeleteClusterStatusMetrics(store *core.StoreInfo) {
+	engine := store.Engine()
+	for _, status := range storeStatuses {
+		clusterStatusGauge.DeleteLabelValues(status, engine, strconv.FormatUint(store.GetID(), 10))
+	}
+}

pkg/statistics/store_collection.go

@@ -52,6 +52,21 @@ const (
 	clusterStatusStorageCapacity = "storage_capacity"
 )
 
+var storeStatuses = []string{
+	clusterStatusStoreUpCount,
+	clusterStatusStoreDisconnectedCount,
+	clusterStatusStoreSlowCount,
+	clusterStatusStoreDownCount,
+	clusterStatusStoreUnhealthCount,
+	clusterStatusStoreOfflineCount,
+	clusterStatusStoreTombstoneCount,
+	clusterStatusStoreLowSpaceCount,
+	clusterStatusStorePreparingCount,
+	clusterStatusStoreServingCount,
+	clusterStatusStoreRemovingCount,
+	clusterStatusStoreRemovedCount,
+}
+
 type storeStatistics struct {
 	opt          config.ConfProvider
 	LabelCounter map[string][]uint64
@@ -129,12 +144,7 @@ func (s *storeStatistics) observe(store *core.StoreInfo) {
 	}
 	storeAddress := store.GetAddress()
 	id := strconv.FormatUint(store.GetID(), 10)
-	var engine string
-	if store.IsTiKV() {
-		engine = core.EngineTiKV
-	} else {
-		engine = core.EngineTiFlash
-	}
+	engine := store.Engine()
 	storeStatusStats := s.observeStoreStatus(store)
 	for statusType, value := range storeStatusStats {
 		clusterStatusGauge.WithLabelValues(statusType, engine, id).Set(value)

server/cluster/cluster.go

@@ -2008,6 +2008,7 @@ func (c *RaftCluster) deleteStore(store *core.StoreInfo) error {
 			return err
 		}
 	}
+	statistics.DeleteClusterStatusMetrics(store)
 	c.DeleteStore(store)
 	return nil
 }