Skip to content

Commit 60312d9

Browse files
klauspostklauspost
committed
Add new AWS checksums
"Reference": https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity-upload.html Validated against AWS S3.
1 parent 68e87b1 commit 60312d9

15 files changed

Lines changed: 299 additions & 14 deletions

‎api-append-object.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,11 @@ func (c *Client) appendObjectDo(ctx context.Context, bucketName, objectName stri
175175
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
176176
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
177177
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
178+
ChecksumMD5: h.Get(ChecksumMD5.Key()),
179+
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
180+
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
181+
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
182+
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
178183
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
179184
}, nil
180185
}

‎api-datatypes.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,11 @@ type UploadInfo struct {
150150
ChecksumSHA1 string
151151
ChecksumSHA256 string
152152
ChecksumCRC64NVME string
153+
ChecksumMD5 string
154+
ChecksumSHA512 string
155+
ChecksumXXHash64 string
156+
ChecksumXXHash3 string
157+
ChecksumXXHash128 string
153158
ChecksumMode string
154159
}
155160

@@ -227,6 +232,11 @@ type ObjectInfo struct {
227232
ChecksumSHA1 string
228233
ChecksumSHA256 string
229234
ChecksumCRC64NVME string
235+
ChecksumMD5 string
236+
ChecksumSHA512 string
237+
ChecksumXXHash64 string
238+
ChecksumXXHash3 string
239+
ChecksumXXHash128 string
230240
ChecksumAlgorithm string
231241
ChecksumMode string `xml:"ChecksumType"`
232242

‎api-get-object-attributes.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,17 @@ type ObjectAttributesResponse struct {
8888
ETag string `xml:",omitempty"`
8989
StorageClass string
9090
ObjectSize int
91-
Checksum struct {
91+
Checksum struct {
9292
ChecksumCRC32 string `xml:",omitempty"`
9393
ChecksumCRC32C string `xml:",omitempty"`
9494
ChecksumCRC64NVME string `xml:",omitempty"`
9595
ChecksumSHA1 string `xml:",omitempty"`
9696
ChecksumSHA256 string `xml:",omitempty"`
97+
ChecksumMD5 string `xml:",omitempty"`
98+
ChecksumSHA512 string `xml:",omitempty"`
99+
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
100+
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
101+
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
97102
ChecksumType string `xml:",omitempty"`
98103
}
99104
ObjectParts struct {
@@ -113,6 +118,11 @@ type ObjectAttributePart struct {
113118
ChecksumCRC64NVME string `xml:",omitempty"`
114119
ChecksumSHA1 string `xml:",omitempty"`
115120
ChecksumSHA256 string `xml:",omitempty"`
121+
ChecksumMD5 string `xml:",omitempty"`
122+
ChecksumSHA512 string `xml:",omitempty"`
123+
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
124+
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
125+
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
116126
PartNumber int
117127
Size int
118128
}
@@ -130,6 +140,11 @@ func (o *ObjectAttributesResponse) ChecksumMap() map[string]string {
130140
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
131141
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
132142
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
143+
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
144+
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
145+
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
146+
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
147+
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
133148
return res
134149
}
135150

@@ -173,6 +188,11 @@ func (o *ObjectAttributesResponse) ChecksumType() ChecksumType {
173188
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
174189
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
175190
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
191+
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
192+
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
193+
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
194+
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
195+
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
176196
if t.IsSet() && o.Checksum.ChecksumType == amzChecksumModeFullObject {
177197
t |= ChecksumFullObject
178198
}

‎api-list.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,11 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
439439
ChecksumSHA1: version.ChecksumSHA1,
440440
ChecksumSHA256: version.ChecksumSHA256,
441441
ChecksumCRC64NVME: version.ChecksumCRC64NVME,
442+
ChecksumMD5: version.ChecksumMD5,
443+
ChecksumSHA512: version.ChecksumSHA512,
444+
ChecksumXXHash64: version.ChecksumXXHash64,
445+
ChecksumXXHash3: version.ChecksumXXHash3,
446+
ChecksumXXHash128: version.ChecksumXXHash128,
442447
}
443448
if !yield(info) {
444449
return false

‎api-put-object-multipart.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,11 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
192192
ChecksumSHA1: part.ChecksumSHA1,
193193
ChecksumSHA256: part.ChecksumSHA256,
194194
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
195+
ChecksumMD5: part.ChecksumMD5,
196+
ChecksumSHA512: part.ChecksumSHA512,
197+
ChecksumXXHash64: part.ChecksumXXHash64,
198+
ChecksumXXHash3: part.ChecksumXXHash3,
199+
ChecksumXXHash128: part.ChecksumXXHash128,
195200
})
196201
}
197202

@@ -353,6 +358,11 @@ func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart
353358
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
354359
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
355360
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
361+
ChecksumMD5: h.Get(ChecksumMD5.Key()),
362+
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
363+
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
364+
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
365+
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
356366
}
357367
objPart.Size = p.size
358368
objPart.PartNumber = p.partNumber
@@ -458,6 +468,11 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
458468
ChecksumCRC32: completeMultipartUploadResult.ChecksumCRC32,
459469
ChecksumCRC32C: completeMultipartUploadResult.ChecksumCRC32C,
460470
ChecksumCRC64NVME: completeMultipartUploadResult.ChecksumCRC64NVME,
471+
ChecksumMD5: completeMultipartUploadResult.ChecksumMD5,
472+
ChecksumSHA512: completeMultipartUploadResult.ChecksumSHA512,
473+
ChecksumXXHash64: completeMultipartUploadResult.ChecksumXXHash64,
474+
ChecksumXXHash3: completeMultipartUploadResult.ChecksumXXHash3,
475+
ChecksumXXHash128: completeMultipartUploadResult.ChecksumXXHash128,
461476
ChecksumMode: completeMultipartUploadResult.ChecksumType,
462477
}, nil
463478
}

‎api-put-object-streaming.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,11 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
259259
ChecksumSHA1: uploadRes.Part.ChecksumSHA1,
260260
ChecksumSHA256: uploadRes.Part.ChecksumSHA256,
261261
ChecksumCRC64NVME: uploadRes.Part.ChecksumCRC64NVME,
262+
ChecksumMD5: uploadRes.Part.ChecksumMD5,
263+
ChecksumSHA512: uploadRes.Part.ChecksumSHA512,
264+
ChecksumXXHash64: uploadRes.Part.ChecksumXXHash64,
265+
ChecksumXXHash3: uploadRes.Part.ChecksumXXHash3,
266+
ChecksumXXHash128: uploadRes.Part.ChecksumXXHash128,
262267
})
263268
}
264269
}
@@ -418,6 +423,11 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
418423
ChecksumSHA1: part.ChecksumSHA1,
419424
ChecksumSHA256: part.ChecksumSHA256,
420425
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
426+
ChecksumMD5: part.ChecksumMD5,
427+
ChecksumSHA512: part.ChecksumSHA512,
428+
ChecksumXXHash64: part.ChecksumXXHash64,
429+
ChecksumXXHash3: part.ChecksumXXHash3,
430+
ChecksumXXHash128: part.ChecksumXXHash128,
421431
})
422432
}
423433

@@ -617,6 +627,11 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
617627
ChecksumSHA1: part.ChecksumSHA1,
618628
ChecksumSHA256: part.ChecksumSHA256,
619629
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
630+
ChecksumMD5: part.ChecksumMD5,
631+
ChecksumSHA512: part.ChecksumSHA512,
632+
ChecksumXXHash64: part.ChecksumXXHash64,
633+
ChecksumXXHash3: part.ChecksumXXHash3,
634+
ChecksumXXHash128: part.ChecksumXXHash128,
620635
})
621636
}
622637

@@ -796,6 +811,11 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
796811
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
797812
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
798813
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
814+
ChecksumMD5: h.Get(ChecksumMD5.Key()),
815+
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
816+
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
817+
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
818+
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
799819
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
800820
}, nil
801821
}

‎api-put-object.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,11 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
502502
ChecksumSHA1: part.ChecksumSHA1,
503503
ChecksumSHA256: part.ChecksumSHA256,
504504
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
505+
ChecksumMD5: part.ChecksumMD5,
506+
ChecksumSHA512: part.ChecksumSHA512,
507+
ChecksumXXHash64: part.ChecksumXXHash64,
508+
ChecksumXXHash3: part.ChecksumXXHash3,
509+
ChecksumXXHash128: part.ChecksumXXHash128,
505510
})
506511
}
507512

‎api-s3-datatypes.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,11 @@ type Version struct {
113113
ChecksumSHA1 string `xml:",omitempty"`
114114
ChecksumSHA256 string `xml:",omitempty"`
115115
ChecksumCRC64NVME string `xml:",omitempty"`
116+
ChecksumMD5 string `xml:",omitempty"`
117+
ChecksumSHA512 string `xml:",omitempty"`
118+
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
119+
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
120+
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
116121
ChecksumAlgorithm string `xml:",omitempty"`
117122
ChecksumType string `xml:",omitempty"`
118123

@@ -298,6 +303,11 @@ type ObjectPart struct {
298303
ChecksumSHA1 string
299304
ChecksumSHA256 string
300305
ChecksumCRC64NVME string
306+
ChecksumMD5 string
307+
ChecksumSHA512 string
308+
ChecksumXXHash64 string
309+
ChecksumXXHash3 string
310+
ChecksumXXHash128 string
301311
}
302312

303313
// Checksum will return the checksum for the given type.
@@ -314,6 +324,16 @@ func (c ObjectPart) Checksum(t ChecksumType) string {
314324
return c.ChecksumSHA256
315325
case t.Is(ChecksumCRC64NVME):
316326
return c.ChecksumCRC64NVME
327+
case t.Is(ChecksumMD5):
328+
return c.ChecksumMD5
329+
case t.Is(ChecksumSHA512):
330+
return c.ChecksumSHA512
331+
case t.Is(ChecksumXXHash64):
332+
return c.ChecksumXXHash64
333+
case t.Is(ChecksumXXHash3):
334+
return c.ChecksumXXHash3
335+
case t.Is(ChecksumXXHash128):
336+
return c.ChecksumXXHash128
317337
}
318338
return ""
319339
}
@@ -383,6 +403,11 @@ type completeMultipartUploadResult struct {
383403
ChecksumSHA1 string
384404
ChecksumSHA256 string
385405
ChecksumCRC64NVME string
406+
ChecksumMD5 string
407+
ChecksumSHA512 string
408+
ChecksumXXHash64 string `xml:"ChecksumXXHASH64"`
409+
ChecksumXXHash3 string `xml:"ChecksumXXHASH3"`
410+
ChecksumXXHash128 string `xml:"ChecksumXXHASH128"`
386411
ChecksumType string
387412
}
388413

@@ -399,6 +424,11 @@ type CompletePart struct {
399424
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
400425
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
401426
ChecksumCRC64NVME string `xml:",omitempty"`
427+
ChecksumMD5 string `xml:",omitempty"`
428+
ChecksumSHA512 string `xml:",omitempty"`
429+
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
430+
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
431+
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
402432
}
403433

404434
// Checksum will return the checksum for the given type.
@@ -415,6 +445,16 @@ func (c CompletePart) Checksum(t ChecksumType) string {
415445
return c.ChecksumSHA256
416446
case t.Is(ChecksumCRC64NVME):
417447
return c.ChecksumCRC64NVME
448+
case t.Is(ChecksumMD5):
449+
return c.ChecksumMD5
450+
case t.Is(ChecksumSHA512):
451+
return c.ChecksumSHA512
452+
case t.Is(ChecksumXXHash64):
453+
return c.ChecksumXXHash64
454+
case t.Is(ChecksumXXHash3):
455+
return c.ChecksumXXHash3
456+
case t.Is(ChecksumXXHash128):
457+
return c.ChecksumXXHash128
418458
}
419459
return ""
420460
}

0 commit comments

Comments
 (0)