diff --git a/CHANGELOG.md b/CHANGELOG.md index 15bbbd8351..b1a3378362 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * [CHANGE] Querier: Mark `-querier.ingester-streaming` flag as deprecated. Now query ingester streaming is always enabled. #5817 * [CHANGE] Compactor/Bucket Store: Added `-blocks-storage.bucket-store.block-discovery-strategy` to configure different block listing strategy. Reverted the current recursive block listing mechanism and use the strategy `Concurrent` as in 1.15. #5828 * [CHANGE] Compactor: Don't halt compactor when overlapped source blocks detected. #5854 +* [CHANGE] S3 Bucket Client: Expose `-blocks-storage.s3.send-content-md5` flag and set default checksum algorithm to MD5. #5870 * [FEATURE] OTLP ingestion experimental. #5813 * [FEATURE] Ingester: Add per-tenant new metric `cortex_ingester_tsdb_data_replay_duration_seconds`. #5477 * [FEATURE] Query Frontend/Scheduler: Add query priority support. #5605 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index bdb418cea1..1aabad8522 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -288,6 +288,12 @@ blocks_storage: # CLI flag: -blocks-storage.s3.bucket-lookup-type [bucket_lookup_type: | default = "auto"] + # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum + # algorithm to verify the provided digest. If false, use CRC32C algorithm + # instead. + # CLI flag: -blocks-storage.s3.send-content-md5 + [send_content_md5: | default = true] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: ] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index a33b129bb9..9ecd0215d1 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -397,6 +397,12 @@ blocks_storage: # CLI flag: -blocks-storage.s3.bucket-lookup-type [bucket_lookup_type: | default = "auto"] + # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum + # algorithm to verify the provided digest. If false, use CRC32C algorithm + # instead. + # CLI flag: -blocks-storage.s3.send-content-md5 + [send_content_md5: | default = true] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: ] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index f28da2b8cf..9b3233270f 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -559,6 +559,12 @@ s3: # CLI flag: -alertmanager-storage.s3.bucket-lookup-type [bucket_lookup_type: | default = "auto"] + # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum + # algorithm to verify the provided digest. If false, use CRC32C algorithm + # instead. + # CLI flag: -alertmanager-storage.s3.send-content-md5 + [send_content_md5: | default = true] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: alertmanager-storage [sse: ] @@ -832,6 +838,12 @@ s3: # CLI flag: -blocks-storage.s3.bucket-lookup-type [bucket_lookup_type: | default = "auto"] + # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum + # algorithm to verify the provided digest. If false, use CRC32C algorithm + # instead. + # CLI flag: -blocks-storage.s3.send-content-md5 + [send_content_md5: | default = true] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: ] @@ -4326,6 +4338,12 @@ s3: # CLI flag: -ruler-storage.s3.bucket-lookup-type [bucket_lookup_type: | default = "auto"] + # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum + # algorithm to verify the provided digest. If false, use CRC32C algorithm + # instead. + # CLI flag: -ruler-storage.s3.send-content-md5 + [send_content_md5: | default = true] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: ruler-storage [sse: ] @@ -4607,6 +4625,12 @@ s3: # CLI flag: -runtime-config.s3.bucket-lookup-type [bucket_lookup_type: | default = "auto"] + # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum + # algorithm to verify the provided digest. If false, use CRC32C algorithm + # instead. + # CLI flag: -runtime-config.s3.send-content-md5 + [send_content_md5: | default = true] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: runtime-config [sse: ] diff --git a/pkg/storage/bucket/s3/bucket_client.go b/pkg/storage/bucket/s3/bucket_client.go index b252035579..f889c12407 100644 --- a/pkg/storage/bucket/s3/bucket_client.go +++ b/pkg/storage/bucket/s3/bucket_client.go @@ -81,13 +81,14 @@ func newS3Config(cfg Config) (s3.Config, error) { } return s3.Config{ - Bucket: cfg.BucketName, - Endpoint: cfg.Endpoint, - Region: cfg.Region, - AccessKey: cfg.AccessKeyID, - SecretKey: cfg.SecretAccessKey.Value, - Insecure: cfg.Insecure, - SSEConfig: sseCfg, + Bucket: cfg.BucketName, + Endpoint: cfg.Endpoint, + Region: cfg.Region, + AccessKey: cfg.AccessKeyID, + SecretKey: cfg.SecretAccessKey.Value, + Insecure: cfg.Insecure, + SSEConfig: sseCfg, + SendContentMd5: cfg.SendContentMd5, HTTPConfig: s3.HTTPConfig{ IdleConnTimeout: model.Duration(cfg.HTTP.IdleConnTimeout), ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout), diff --git a/pkg/storage/bucket/s3/config.go b/pkg/storage/bucket/s3/config.go index 840bf48137..e138079139 100644 --- a/pkg/storage/bucket/s3/config.go +++ b/pkg/storage/bucket/s3/config.go @@ -66,6 +66,7 @@ type Config struct { Insecure bool `yaml:"insecure"` SignatureVersion string `yaml:"signature_version"` BucketLookupType string `yaml:"bucket_lookup_type"` + SendContentMd5 bool `yaml:"send_content_md5"` SSE SSEConfig `yaml:"sse"` HTTP HTTPConfig `yaml:"http"` @@ -86,6 +87,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.BoolVar(&cfg.Insecure, prefix+"s3.insecure", false, "If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.") f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", "))) f.StringVar(&cfg.BucketLookupType, prefix+"s3.bucket-lookup-type", BucketAutoLookup, fmt.Sprintf("The s3 bucket lookup style. Supported values are: %s.", strings.Join(supportedBucketLookupTypes, ", "))) + f.BoolVar(&cfg.SendContentMd5, prefix+"s3.send-content-md5", true, "If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum algorithm to verify the provided digest. If false, use CRC32C algorithm instead.") cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f) cfg.HTTP.RegisterFlagsWithPrefix(prefix, f) } diff --git a/pkg/storage/bucket/s3/config_test.go b/pkg/storage/bucket/s3/config_test.go index 626610efa2..c61bfe9c37 100644 --- a/pkg/storage/bucket/s3/config_test.go +++ b/pkg/storage/bucket/s3/config_test.go @@ -18,6 +18,7 @@ import ( var defaultConfig = Config{ SignatureVersion: SignatureVersionV4, BucketLookupType: BucketAutoLookup, + SendContentMd5: true, HTTP: HTTPConfig{ Config: bucket_http.Config{ IdleConnTimeout: 90 * time.Second, @@ -78,6 +79,7 @@ http: Insecure: true, SignatureVersion: "test-signature-version", BucketLookupType: BucketVirtualHostLookup, + SendContentMd5: true, SSE: SSEConfig{ Type: "test-type", KMSKeyID: "test-kms-key-id",