Skip to content

Commit 75220b7

Browse files
committed
fixes issues after rebase
1 parent 1f137d3 commit 75220b7

File tree

3 files changed

+48
-21
lines changed

3 files changed

+48
-21
lines changed

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import software.amazon.awssdk.core.interceptor.Context;
3030
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
3131
import software.amazon.awssdk.http.SdkHttpRequest;
32+
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
33+
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
3234

3335
import org.slf4j.Logger;
3436
import org.slf4j.LoggerFactory;
@@ -260,7 +262,8 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl {
260262

261263
/**
262264
* Attach Range of data for GetObject Request.
263-
* @param request given get object request
265+
* @param request the sdk request to be modified
266+
* @param executionAttributes execution attributes for this request
264267
*/
265268
private void attachRangeFromRequest(SdkHttpRequest request,
266269
ExecutionAttributes executionAttributes) {
@@ -370,12 +373,13 @@ public void set(final String key, final String value) {
370373
public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
371374
ExecutionAttributes executionAttributes) {
372375
SdkHttpRequest httpRequest = context.httpRequest();
376+
SdkRequest sdkRequest = context.request();
373377

374-
// attach range for GetObject requests
375-
attachRangeFromRequest(httpRequest, executionAttributes);
378+
// attach range for GetObject requests
379+
attachRangeFromRequest(httpRequest, executionAttributes);
376380

377-
// for delete op, attach the number of files to delete
378-
attachDeleteKeySizeAttribute(request);
381+
// for delete op, attach the number of files to delete
382+
attachDeleteKeySizeAttribute(sdkRequest);
379383

380384
// build the referrer header
381385
final String header = referrer.buildHttpReferrer();
@@ -397,9 +401,9 @@ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
397401
}
398402

399403
// now see if the request is actually a blocked multipart request
400-
if (!isMultipartUploadEnabled && isRequestMultipartIO(httpRequest)) {
404+
if (!isMultipartUploadEnabled && isRequestMultipartIO(sdkRequest)) {
401405
throw new AuditOperationRejectedException("Multipart IO request "
402-
+ httpRequest + " rejected " + header);
406+
+ sdkRequest + " rejected " + header);
403407
}
404408

405409
return httpRequest;
@@ -409,16 +413,16 @@ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
409413
* For delete requests, attach delete key size as a referrer attribute.
410414
*
411415
* @param request the request object.
412-
* @param <T> type of the request.
413416
*/
414-
private <T extends AmazonWebServiceRequest> void attachDeleteKeySizeAttribute(T request) {
417+
private void attachDeleteKeySizeAttribute(SdkRequest request) {
418+
415419
if (request instanceof DeleteObjectsRequest) {
416-
int keySize = ((DeleteObjectsRequest) request).getKeys().size();
417-
this.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
420+
int keySize = ((DeleteObjectsRequest) request).delete().objects().size();
421+
referrer.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
418422
} else if (request instanceof DeleteObjectRequest) {
419-
String key = ((DeleteObjectRequest) request).getKey();
423+
String key = ((DeleteObjectRequest) request).key();
420424
if (key != null && key.length() > 0) {
421-
this.set(DELETE_KEYS_SIZE, "1");
425+
referrer.set(DELETE_KEYS_SIZE, "1");
422426
}
423427
}
424428
}

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,11 @@
3434
import software.amazon.awssdk.core.interceptor.InterceptorContext;
3535
import software.amazon.awssdk.http.SdkHttpMethod;
3636
import software.amazon.awssdk.http.SdkHttpRequest;
37+
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
3738
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
3839
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
40+
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
41+
3942
import org.junit.After;
4043
import org.junit.Before;
4144
import org.slf4j.Logger;
@@ -288,15 +291,31 @@ protected void assertMapNotContains(final Map<String, String> params, final Stri
288291
* @param keys keys to be provided in the bulk delete request.
289292
* @return a processed request.
290293
*/
291-
protected DeleteObjectsRequest headForBulkDelete(String... keys) {
294+
protected SdkHttpRequest headForBulkDelete(String... keys) {
292295
if (keys == null || keys.length == 0) {
293296
return null;
294297
}
295-
List<DeleteObjectsRequest.KeyVersion> keysToDelete = Arrays
298+
299+
List<ObjectIdentifier> keysToDelete = Arrays
296300
.stream(keys)
297-
.map(DeleteObjectsRequest.KeyVersion::new)
301+
.map(key -> ObjectIdentifier.builder().key(key).build())
298302
.collect(Collectors.toList());
299-
return manager.beforeExecution(requestFactory.newBulkDeleteRequest(keysToDelete));
303+
304+
ExecutionAttributes executionAttributes = ExecutionAttributes.builder().build();
305+
306+
SdkHttpRequest.Builder httpRequestBuilder =
307+
SdkHttpRequest.builder().uri(URI.create("https://test")).method(SdkHttpMethod.POST);
308+
309+
DeleteObjectsRequest deleteObjectsRequest =
310+
requestFactory.newBulkDeleteRequestBuilder(keysToDelete).build();
311+
312+
InterceptorContext context = InterceptorContext.builder()
313+
.request(deleteObjectsRequest)
314+
.httpRequest(httpRequestBuilder.build())
315+
.build();
316+
317+
manager.beforeExecution(context, executionAttributes);
318+
return manager.modifyHttpRequest(context, executionAttributes);
300319
}
301320

302321
}

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -350,16 +350,20 @@ public void testGetObjectWithoutRange() throws Throwable {
350350
public void testHttpReferrerForBulkDelete() throws Throwable {
351351
AuditSpan span = span();
352352
long ts = span.getTimestamp();
353-
DeleteObjectsRequest request = headForBulkDelete(
353+
SdkHttpRequest request = headForBulkDelete(
354354
"key_01",
355355
"key_02",
356356
"key_03");
357-
Map<String, String> headers
358-
= request.getCustomRequestHeaders();
357+
Map<String, List<String>> headers
358+
= request.headers();
359359
assertThat(headers)
360360
.describedAs("Custom headers")
361361
.containsKey(HEADER_REFERRER);
362-
String header = headers.get(HEADER_REFERRER);
362+
List<String> headerValues = headers.get(HEADER_REFERRER);
363+
assertThat(headerValues)
364+
.describedAs("Multiple referrer headers")
365+
.hasSize(1);
366+
String header = headerValues.get(0);
363367
LOG.info("Header is {}", header);
364368
Map<String, String> params
365369
= HttpReferrerAuditHeader.extractQueryParameters(header);

0 commit comments

Comments
 (0)