If ingester.max-transfer-retries is set to 0, hand-over attempts are … (#1777)

* If ingester.max-transfer-retries is set to 0, hand-over attempts are disabled.

Due to interaction with BackOffConfig.MaxRetries, zero value meant infinite number
of attempts, which is hardly ever a desired behaviour (with no ingester waiting to
accept a transfer, this would never end).

Fixed #1771.

Signed-off-by: Peter Štibraný <[email protected]>

* Fixed typo
Signed-off-by: Peter Štibraný <[email protected]>

* Fix ingester.max-transfer-retries argument name.

Signed-off-by: Peter Štibraný <[email protected]>

Author: pstibrany

CHANGELOG.md

@@ -5,6 +5,7 @@
 * [CHANGE] The frontend component has been refactored to be easier to re-use. When upgrading the frontend, cache entries will be discarded and re-created with the new protobuf schema. #1734
 * [CHANGE] Remove direct DB/API access from the ruler
 * [CHANGE] Removed `Delta` encoding. Any old chunks with `Delta` encoding cannot be read anymore. If `ingester.chunk-encoding` is set to `Delta` the ingester will fail to start. #1706
+* [CHANGE] Setting `-ingester.max-transfer-retries` to 0 now disables hand-over when ingester is shutting down. Previously, zero meant infinite number of attempts. #1771
 * [FEATURE] Global limit on the max series per user and metric #1760
   * `-ingester.max-global-series-per-user`
   * `-ingester.max-global-series-per-metric`

docs/arguments.md

@@ -228,9 +228,9 @@ It also talks to a KVStore and has it's own copies of the same flags used by the
 
    How long to wait in PENDING state during the [hand-over process](ingester-handover.md). (default 0s)
 
-- `-ingester.ingester.max-transfer-retries`
+- `-ingester.max-transfer-retries`
 
-   How many times a LEAVING ingester tries to find a PENDING ingester during the [hand-over process](ingester-handover.md). Each attempt takes a second or so. (default 10)
+   How many times a LEAVING ingester tries to find a PENDING ingester during the [hand-over process](ingester-handover.md). Each attempt takes a second or so. Negative value or zero disables hand-over process completely. (default 10)
 
 - `-ingester.normalise-tokens`
 

pkg/ingester/ingester.go

@@ -102,7 +102,7 @@ func newIngesterMetrics(r prometheus.Registerer) *ingesterMetrics {
 type Config struct {
 	LifecyclerConfig ring.LifecyclerConfig `yaml:"lifecycler,omitempty"`
 
-	// Config for transferring chunks.
+	// Config for transferring chunks. Zero or negative = no retries.
 	MaxTransferRetries int `yaml:"max_transfer_retries,omitempty"`
 
 	// Config for chunk flushing.
@@ -134,7 +134,7 @@ type Config struct {
 func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	cfg.LifecyclerConfig.RegisterFlags(f)
 
-	f.IntVar(&cfg.MaxTransferRetries, "ingester.max-transfer-retries", 10, "Number of times to try and transfer chunks before falling back to flushing.")
+	f.IntVar(&cfg.MaxTransferRetries, "ingester.max-transfer-retries", 10, "Number of times to try and transfer chunks before falling back to flushing. Negative value or zero disables hand-over.")
 	f.DurationVar(&cfg.FlushCheckPeriod, "ingester.flush-period", 1*time.Minute, "Period with which to attempt to flush chunks.")
 	f.DurationVar(&cfg.RetainPeriod, "ingester.retain-period", 5*time.Minute, "Period chunks will remain in memory after flushing.")
 	f.DurationVar(&cfg.FlushOpTimeout, "ingester.flush-op-timeout", 1*time.Minute, "Timeout for individual flush operations.")

pkg/ingester/lifecycle_test.go

@@ -44,7 +44,7 @@ func defaultIngesterTestConfig() Config {
 	cfg.LifecyclerConfig.Addr = "localhost"
 	cfg.LifecyclerConfig.ID = "localhost"
 	cfg.LifecyclerConfig.FinalSleep = 0
-	cfg.MaxTransferRetries = -1
+	cfg.MaxTransferRetries = 0
 	return cfg
 }
 

pkg/ingester/transfer.go

@@ -349,7 +349,7 @@ func fromWireChunks(wireChunks []client.Chunk) ([]*desc, error) {
 // TransferOut finds an ingester in PENDING state and transfers our chunks to it.
 // Called as part of the ingester shutdown process.
 func (i *Ingester) TransferOut(ctx context.Context) error {
-	if i.cfg.MaxTransferRetries < 0 {
+	if i.cfg.MaxTransferRetries <= 0 {
 		return fmt.Errorf("transfers disabled")
 	}
 	backoff := util.NewBackoff(ctx, util.BackoffConfig{