Skip to content

Commit 347cc9f

Browse files
jxhan3jxhan3
committed
[S3AG] make plugin optional
1 parent 01052d0 commit 347cc9f

File tree

4 files changed

+93
-60
lines changed

4 files changed

+93
-60
lines changed

‎hadoop-tools/hadoop-aws/pom.xml

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -464,16 +464,6 @@
464464
<bannedImport>org.apache.hadoop.mapred.**</bannedImport>
465465
</bannedImports>
466466
</restrictImports>
467-
<restrictImports>
468-
<includeTestCode>false</includeTestCode>
469-
<reason>Restrict S3AG plugin class imports to default client factory</reason>
470-
<exclusions>
471-
<exclusion>org.apache.hadoop.fs.s3a.DefaultS3ClientFactory</exclusion>
472-
</exclusions>
473-
<bannedImports>
474-
<bannedImport>software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin</bannedImport>
475-
</bannedImports>
476-
</restrictImports>
477467
</rules>
478468
</configuration>
479469
</execution>

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java

Lines changed: 29 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@
1919
package org.apache.hadoop.fs.s3a;
2020

2121
import java.io.IOException;
22+
import java.lang.reflect.Method;
2223
import java.net.URI;
2324
import java.net.URISyntaxException;
2425

2526
import org.apache.hadoop.classification.VisibleForTesting;
2627
import org.apache.hadoop.fs.s3a.impl.AWSClientConfig;
27-
import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
2828
import org.slf4j.Logger;
2929
import org.slf4j.LoggerFactory;
3030

@@ -38,7 +38,6 @@
3838
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
3939
import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
4040
import software.amazon.awssdk.regions.Region;
41-
import software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin;
4241
import software.amazon.awssdk.services.s3.S3AsyncClient;
4342
import software.amazon.awssdk.services.s3.S3BaseClientBuilder;
4443
import software.amazon.awssdk.services.s3.S3Client;
@@ -56,7 +55,6 @@
5655

5756
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
5857
import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESS_GRANTS_ENABLED;
59-
import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED;
6058
import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_DEFAULT_REGION;
6159
import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
6260
import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
@@ -68,7 +66,6 @@
6866
import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3;
6967
import static org.apache.hadoop.fs.s3a.auth.SignerFactory.createHttpSigner;
7068
import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER;
71-
import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable;
7269
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AUTH_SCHEME_AWS_SIGV_4;
7370
import static org.apache.hadoop.util.Preconditions.checkArgument;
7471

@@ -119,34 +116,8 @@ public class DefaultS3ClientFactory extends Configured
119116
public static final String ERROR_ENDPOINT_WITH_FIPS =
120117
"An endpoint cannot set when " + FIPS_ENDPOINT + " is true";
121118

122-
private static final String S3AG_PLUGIN_CLASSNAME =
123-
"software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin";
124-
125-
/**
126-
* S3 Access Grants plugin availability.
127-
*/
128-
private static final boolean S3AG_PLUGIN_FOUND = checkForS3AGPlugin();
129-
130-
private static boolean checkForS3AGPlugin() {
131-
try {
132-
ClassLoader cl = DefaultS3ClientFactory.class.getClassLoader();
133-
cl.loadClass(S3AG_PLUGIN_CLASSNAME);
134-
LOG.debug("S3AG plugin class {} found", S3AG_PLUGIN_CLASSNAME);
135-
return true;
136-
} catch (Exception e) {
137-
LOG.debug("S3AG plugin class {} not found", S3AG_PLUGIN_CLASSNAME, e);
138-
return false;
139-
}
140-
}
141-
142-
/**
143-
* Is the Encryption client available?
144-
* @return true if it was found in the classloader
145-
*/
146-
private static synchronized boolean isS3AGPluginAvailable() {
147-
return S3AG_PLUGIN_FOUND;
148-
}
149-
119+
private static final String S3AG_UTIL_CLASSNAME =
120+
"org.apache.hadoop.fs.s3a.tools.S3AccessGrantsUtil";
150121
@Override
151122
public S3Client createS3Client(
152123
final URI uri,
@@ -441,19 +412,32 @@ private static Region getS3RegionFromEndpoint(final String endpoint,
441412
public static <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> void
442413
applyS3AccessGrantsConfigurations(BuilderT builder, Configuration conf) {
443414
boolean s3agEnabled = conf.getBoolean(AWS_S3_ACCESS_GRANTS_ENABLED, false);
444-
if (s3agEnabled) {
445-
if (isS3AGPluginAvailable()) {
446-
boolean s3agFallbackEnabled = conf.getBoolean(
447-
AWS_S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED, false);
448-
S3AccessGrantsPlugin accessGrantsPlugin =
449-
S3AccessGrantsPlugin.builder().enableFallback(s3agFallbackEnabled).build();
450-
builder.addPlugin(accessGrantsPlugin);
451-
LOG_EXACTLY_ONCE.info("s3ag plugin is added to s3 client with fallback: {}", s3agFallbackEnabled);
452-
} else {
453-
LOG_EXACTLY_ONCE.warn("s3ag plugin is not available.");
454-
}
455-
} else {
456-
LOG_EXACTLY_ONCE.debug("s3ag plugin is not added to s3 client.");
415+
if (!s3agEnabled){
416+
LOG_EXACTLY_ONCE.debug("s3ag plugin is not enabled.");
417+
return;
418+
}
419+
try {
420+
Class s3agUtil = Class.forName(S3AG_UTIL_CLASSNAME);
421+
Class[] argTypes = new Class[2];
422+
argTypes[0] = S3BaseClientBuilder.class;
423+
argTypes[1] = Configuration.class;
424+
Method applyS3agConfig =
425+
s3agUtil.getMethod("applyS3AccessGrantsConfigurations", S3BaseClientBuilder.class, Configuration.class);
426+
applyS3agConfig.invoke(null, builder, conf);
427+
} catch (ClassNotFoundException e) {
428+
LOG_EXACTLY_ONCE.debug(
429+
"Class {} is not found exception: {}.",
430+
S3AG_UTIL_CLASSNAME,
431+
e.getStackTrace()
432+
);
433+
} catch (Exception e) {
434+
LOG_EXACTLY_ONCE.debug("{} exception: {})", e.getClass(), e.getStackTrace());
435+
} catch (NoClassDefFoundError e) {
436+
LOG_EXACTLY_ONCE.debug(
437+
"Class {} is not found error: ",
438+
S3AG_UTIL_CLASSNAME,
439+
e.getStackTrace()
440+
);
457441
}
458442
}
459443
}

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/S3AccessGrantsUtil.java

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package org.apache.hadoop.fs.s3a.tools;
2+
3+
import org.apache.hadoop.conf.Configuration;
4+
import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;
5+
import org.apache.hadoop.fs.store.LogExactlyOnce;
6+
import org.slf4j.Logger;
7+
import org.slf4j.LoggerFactory;
8+
import software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin;
9+
import software.amazon.awssdk.services.s3.S3BaseClientBuilder;
10+
11+
import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED;
12+
13+
public class S3AccessGrantsUtil {
14+
15+
protected static final Logger LOG =
16+
LoggerFactory.getLogger(S3AccessGrantsUtil.class);
17+
18+
private static final LogExactlyOnce LOG_EXACTLY_ONCE = new LogExactlyOnce(LOG);
19+
private static final String S3AG_PLUGIN_CLASSNAME =
20+
"software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin";
21+
22+
/**
23+
* S3 Access Grants plugin availability.
24+
*/
25+
private static final boolean S3AG_PLUGIN_FOUND = checkForS3AGPlugin();
26+
27+
private static boolean checkForS3AGPlugin() {
28+
try {
29+
ClassLoader cl = DefaultS3ClientFactory.class.getClassLoader();
30+
cl.loadClass(S3AG_PLUGIN_CLASSNAME);
31+
LOG.debug("S3AG plugin class {} found", S3AG_PLUGIN_CLASSNAME);
32+
return true;
33+
} catch (Exception e) {
34+
LOG.debug("S3AG plugin class {} not found", S3AG_PLUGIN_CLASSNAME, e);
35+
return false;
36+
}
37+
}
38+
39+
/**
40+
* Is the S3AG plugin available?
41+
* @return true if it was found in the classloader
42+
*/
43+
private static synchronized boolean isS3AGPluginAvailable() {
44+
return S3AG_PLUGIN_FOUND;
45+
}
46+
47+
public static <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> void
48+
applyS3AccessGrantsConfigurations(BuilderT builder, Configuration conf) {
49+
if (isS3AGPluginAvailable()) {
50+
boolean s3agFallbackEnabled = conf.getBoolean(
51+
AWS_S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED, false);
52+
S3AccessGrantsPlugin accessGrantsPlugin =
53+
S3AccessGrantsPlugin.builder().enableFallback(s3agFallbackEnabled).build();
54+
builder.addPlugin(accessGrantsPlugin);
55+
LOG_EXACTLY_ONCE.info("s3ag plugin is added to s3 client with fallback: {}", s3agFallbackEnabled);
56+
} else {
57+
LOG_EXACTLY_ONCE.warn("s3ag plugin is not available.");
58+
}
59+
}
60+
}

‎hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AccessGrantConfiguration.java

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,7 @@
1919
package org.apache.hadoop.fs.s3a;
2020

2121
import org.apache.hadoop.conf.Configuration;
22-
import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
2322
import org.apache.hadoop.test.AbstractHadoopTestBase;
24-
import org.junit.Assert;
2523
import org.junit.Test;
2624

2725
import software.amazon.awssdk.services.s3.S3AsyncClient;
@@ -79,9 +77,10 @@ private Configuration createConfig(boolean isDefault, boolean s3agEnabled) {
7977
applyVerifyS3AGPlugin(BuilderT builder, boolean isDefault, boolean enabled) {
8078
DefaultS3ClientFactory.applyS3AccessGrantsConfigurations(builder, createConfig(isDefault, enabled));
8179
if (enabled){
82-
assertEquals(builder.plugins().size(), 1);
83-
assertEquals(builder.plugins().get(0).getClass().getName(),
84-
"software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin");
80+
assertEquals(1, builder.plugins().size());
81+
assertEquals("software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin",
82+
builder.plugins().get(0).getClass().getName()
83+
);
8584
}
8685
else {
8786
assertEquals(builder.plugins().size(), 0);

0 commit comments

Comments
 (0)