Skip to content

Commit 6cb2e86

Browse files
pan3793pan3793
authored
HADOOP-19367. Fix setting final field value on Java 17 (#7228)
1 parent 7f49190 commit 6cb2e86

File tree

9 files changed

+60
-142
lines changed

9 files changed

+60
-142
lines changed

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/ReflectionUtils.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
package org.apache.hadoop.test;
1919

2020
import java.lang.reflect.Field;
21+
import java.lang.reflect.Method;
22+
import java.lang.reflect.Modifier;
2123

2224
public final class ReflectionUtils {
2325
private ReflectionUtils() {}
@@ -48,4 +50,27 @@ public static String getStringValueOfField(Field f) throws IllegalAccessExceptio
4850
return null;
4951
}
5052
}
53+
54+
public static <T> void setFinalField(
55+
Class<T> type, final T obj, final String fieldName, Object value)
56+
throws ReflectiveOperationException {
57+
Field f = type.getDeclaredField(fieldName);
58+
f.setAccessible(true);
59+
Field modifiersField = ReflectionUtils.getModifiersField();
60+
modifiersField.setAccessible(true);
61+
modifiersField.setInt(f, f.getModifiers() & ~Modifier.FINAL);
62+
f.set(obj, value);
63+
}
64+
65+
public static Field getModifiersField() throws ReflectiveOperationException {
66+
Method getDeclaredFields0 = Class.class.getDeclaredMethod("getDeclaredFields0", boolean.class);
67+
getDeclaredFields0.setAccessible(true);
68+
Field[] fields = (Field[]) getDeclaredFields0.invoke(Field.class, false);
69+
for (Field each : fields) {
70+
if ("modifiers".equals(each.getName())) {
71+
return each;
72+
}
73+
}
74+
throw new UnsupportedOperationException();
75+
}
5176
}

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,6 @@
4646
import java.io.FileNotFoundException;
4747
import java.io.IOException;
4848
import java.io.InputStreamReader;
49-
import java.lang.reflect.Field;
50-
import java.lang.reflect.Modifier;
5149
import java.net.InetSocketAddress;
5250
import java.net.URI;
5351
import java.net.UnknownHostException;
@@ -90,6 +88,7 @@
9088
import org.apache.hadoop.io.IOUtils;
9189
import org.apache.hadoop.security.UserGroupInformation;
9290
import org.apache.hadoop.test.GenericTestUtils;
91+
import org.apache.hadoop.test.ReflectionUtils;
9392
import org.apache.hadoop.util.Time;
9493
import org.junit.Assert;
9594
import org.junit.Test;
@@ -715,7 +714,7 @@ public void testFileCreationError3() throws IOException {
715714
*/
716715
@Test
717716
public void testFileCreationNamenodeRestart()
718-
throws IOException, NoSuchFieldException, IllegalAccessException {
717+
throws IOException, ReflectiveOperationException {
719718
Configuration conf = new HdfsConfiguration();
720719
final int MAX_IDLE_TIME = 2000; // 2s
721720
conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME);
@@ -812,20 +811,13 @@ public void testFileCreationNamenodeRestart()
812811

813812
// instruct the dfsclient to use a new filename when it requests
814813
// new blocks for files that were renamed.
815-
DFSOutputStream dfstream = (DFSOutputStream)
816-
(stm.getWrappedStream());
814+
DFSOutputStream dfstream = (DFSOutputStream) (stm.getWrappedStream());
817815

818-
Field f = DFSOutputStream.class.getDeclaredField("src");
819-
Field modifiersField = Field.class.getDeclaredField("modifiers");
820-
modifiersField.setAccessible(true);
821-
modifiersField.setInt(f, f.getModifiers() & ~Modifier.FINAL);
822-
f.setAccessible(true);
823-
824-
f.set(dfstream, file1.toString());
816+
ReflectionUtils.setFinalField(DFSOutputStream.class, dfstream, "src", file1.toString());
825817
dfstream = (DFSOutputStream) (stm3.getWrappedStream());
826-
f.set(dfstream, file3new.toString());
818+
ReflectionUtils.setFinalField(DFSOutputStream.class, dfstream, "src", file3new.toString());
827819
dfstream = (DFSOutputStream) (stm4.getWrappedStream());
828-
f.set(dfstream, file4new.toString());
820+
ReflectionUtils.setFinalField(DFSOutputStream.class, dfstream, "src", file4new.toString());
829821

830822
// write 1 byte to file. This should succeed because the
831823
// namenode should have persisted leases.

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStoragePolicyPermissionSettings.java

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@
2121
import static org.junit.Assert.assertNotEquals;
2222

2323
import java.io.IOException;
24-
import java.lang.reflect.Field;
25-
import java.lang.reflect.Modifier;
2624

2725
import org.apache.hadoop.conf.Configuration;
2826
import org.apache.hadoop.fs.FileSystem;
@@ -34,6 +32,7 @@
3432
import org.apache.hadoop.security.AccessControlException;
3533
import org.apache.hadoop.security.UserGroupInformation;
3634
import org.apache.hadoop.test.LambdaTestUtils;
35+
import org.apache.hadoop.test.ReflectionUtils;
3736
import org.junit.AfterClass;
3837
import org.junit.BeforeClass;
3938
import org.junit.Test;
@@ -77,22 +76,15 @@ public static void clusterShutdown() throws IOException {
7776
}
7877
}
7978

80-
private void setFSNameSystemFinalField(String field, boolean value)
81-
throws NoSuchFieldException, IllegalAccessException {
82-
Field f = FSNamesystem.class.getDeclaredField(field);
83-
f.setAccessible(true);
84-
Field modifiersField = Field.class.getDeclaredField("modifiers");
85-
modifiersField.setAccessible(true);
86-
modifiersField.setInt(f, f.getModifiers() & ~Modifier.FINAL);
87-
f.set(cluster.getNamesystem(), value);
88-
}
89-
9079
private void setStoragePolicyPermissions(boolean isStoragePolicyEnabled,
9180
boolean isStoragePolicySuperuserOnly)
92-
throws NoSuchFieldException, IllegalAccessException {
93-
setFSNameSystemFinalField("isStoragePolicyEnabled", isStoragePolicyEnabled);
94-
setFSNameSystemFinalField("isStoragePolicySuperuserOnly",
95-
isStoragePolicySuperuserOnly);
81+
throws ReflectiveOperationException {
82+
ReflectionUtils.setFinalField(
83+
FSNamesystem.class, cluster.getNamesystem(),
84+
"isStoragePolicyEnabled", isStoragePolicyEnabled);
85+
ReflectionUtils.setFinalField(
86+
FSNamesystem.class, cluster.getNamesystem(),
87+
"isStoragePolicySuperuserOnly", isStoragePolicySuperuserOnly);
9688
}
9789

9890
@Test

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import java.io.FileNotFoundException;
2222
import java.io.FilterOutputStream;
2323
import java.io.IOException;
24-
import java.lang.reflect.Field;
2524
import java.util.EnumSet;
2625
import java.util.UUID;
2726

@@ -40,6 +39,7 @@
4039
import org.apache.hadoop.fs.permission.FsAction;
4140
import org.apache.hadoop.fs.permission.FsPermission;
4241
import org.apache.hadoop.test.GenericTestUtils;
42+
import org.apache.hadoop.test.ReflectionUtils;
4343

4444
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
4545
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
@@ -395,7 +395,9 @@ public void testNegativeScenariosForCreateOverwriteDisabled()
395395
fs.getAbfsStore().getAbfsConfiguration());
396396

397397
AzureBlobFileSystemStore abfsStore = fs.getAbfsStore();
398-
abfsStore = setAzureBlobSystemStoreField(abfsStore, "client", mockClient);
398+
399+
ReflectionUtils.setFinalField(AzureBlobFileSystemStore.class, abfsStore, "client", mockClient);
400+
399401
boolean isNamespaceEnabled = abfsStore
400402
.getIsNamespaceEnabled(getTestTracingContext(fs, false));
401403

@@ -486,22 +488,6 @@ public void testNegativeScenariosForCreateOverwriteDisabled()
486488
validateCreateFileException(AbfsRestOperationException.class, abfsStore);
487489
}
488490

489-
private AzureBlobFileSystemStore setAzureBlobSystemStoreField(
490-
final AzureBlobFileSystemStore abfsStore,
491-
final String fieldName,
492-
Object fieldObject) throws Exception {
493-
494-
Field abfsClientField = AzureBlobFileSystemStore.class.getDeclaredField(
495-
fieldName);
496-
abfsClientField.setAccessible(true);
497-
Field modifiersField = Field.class.getDeclaredField("modifiers");
498-
modifiersField.setAccessible(true);
499-
modifiersField.setInt(abfsClientField,
500-
abfsClientField.getModifiers() & ~java.lang.reflect.Modifier.FINAL);
501-
abfsClientField.set(abfsStore, fieldObject);
502-
return abfsStore;
503-
}
504-
505491
private <E extends Throwable> void validateCreateFileException(final Class<E> exceptionClass, final AzureBlobFileSystemStore abfsStore)
506492
throws Exception {
507493
FsPermission permission = new FsPermission(FsAction.ALL, FsAction.ALL,

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,12 @@
4141
import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
4242
import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
4343
import org.apache.hadoop.fs.azurebfs.services.TestAbfsPerfTracker;
44-
import org.apache.hadoop.fs.azurebfs.utils.TestMockHelpers;
4544
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
4645
import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
4746
import org.apache.hadoop.fs.FileAlreadyExistsException;
4847
import org.apache.hadoop.fs.FileStatus;
4948
import org.apache.hadoop.fs.Path;
49+
import org.apache.hadoop.test.ReflectionUtils;
5050

5151
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
5252
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
@@ -250,9 +250,9 @@ public void testDeleteIdempotencyTriggerHttp404() throws Exception {
250250
fs.getAbfsStore().getClient(),
251251
this.getConfiguration());
252252
AzureBlobFileSystemStore mockStore = mock(AzureBlobFileSystemStore.class);
253-
mockStore = TestMockHelpers.setClassField(AzureBlobFileSystemStore.class, mockStore,
253+
ReflectionUtils.setFinalField(AzureBlobFileSystemStore.class, mockStore,
254254
"client", mockClient);
255-
mockStore = TestMockHelpers.setClassField(AzureBlobFileSystemStore.class,
255+
ReflectionUtils.setFinalField(AzureBlobFileSystemStore.class,
256256
mockStore,
257257
"abfsPerfTracker",
258258
TestAbfsPerfTracker.getAPerfTrackerInstance(this.getConfiguration()));

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java

Lines changed: 9 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
package org.apache.hadoop.fs.azurebfs.services;
2020

2121
import java.io.IOException;
22-
import java.lang.reflect.Field;
2322
import java.net.ProtocolException;
2423
import java.net.URI;
2524
import java.net.URISyntaxException;
@@ -29,17 +28,18 @@
2928
import java.util.Random;
3029
import java.util.regex.Pattern;
3130

32-
import org.apache.hadoop.fs.azurebfs.AbfsCountersImpl;
3331
import org.assertj.core.api.Assertions;
3432
import org.junit.Assume;
3533
import org.junit.Test;
3634
import org.junit.runner.RunWith;
3735
import org.junit.runners.Parameterized;
3836
import org.mockito.Mockito;
3937

38+
import org.apache.hadoop.conf.Configuration;
4039
import org.apache.hadoop.fs.FileSystem;
4140
import org.apache.hadoop.fs.Path;
4241
import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
42+
import org.apache.hadoop.fs.azurebfs.AbfsCountersImpl;
4343
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
4444
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
4545
import org.apache.hadoop.fs.azurebfs.TestAbfsConfigurationFieldsValidation;
@@ -49,11 +49,11 @@
4949
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
5050
import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters;
5151
import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
52-
import org.apache.hadoop.conf.Configuration;
5352
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
5453
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
5554
import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat;
5655
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
56+
import org.apache.hadoop.test.ReflectionUtils;
5757
import org.apache.http.HttpResponse;
5858

5959
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
@@ -423,50 +423,30 @@ public static AbfsClient getMockAbfsClient(AbfsClient baseAbfsClientInstance,
423423
Mockito.doReturn(baseAbfsClientInstance.getAbfsApacheHttpClient()).when(client).getAbfsApacheHttpClient();
424424

425425
// override baseurl
426-
client = ITestAbfsClient.setAbfsClientField(client, "abfsConfiguration",
427-
abfsConfig);
426+
ReflectionUtils.setFinalField(AbfsClient.class, client, "abfsConfiguration", abfsConfig);
428427

429428
// override baseurl
430-
client = ITestAbfsClient.setAbfsClientField(client, "baseUrl",
431-
baseAbfsClientInstance.getBaseUrl());
429+
ReflectionUtils.setFinalField(AbfsClient.class, client, "baseUrl", baseAbfsClientInstance.getBaseUrl());
432430

433431
// override xMsVersion
434-
client = ITestAbfsClient.setAbfsClientField(client, "xMsVersion",
435-
baseAbfsClientInstance.getxMsVersion());
432+
ReflectionUtils.setFinalField(AbfsClient.class, client, "xMsVersion", baseAbfsClientInstance.getxMsVersion());
436433

437434
// override auth provider
438435
if (currentAuthType == AuthType.SharedKey) {
439-
client = ITestAbfsClient.setAbfsClientField(client, "sharedKeyCredentials",
440-
new SharedKeyCredentials(
436+
ReflectionUtils.setFinalField(AbfsClient.class, client, "sharedKeyCredentials", new SharedKeyCredentials(
441437
abfsConfig.getAccountName().substring(0,
442438
abfsConfig.getAccountName().indexOf(DOT)),
443439
abfsConfig.getStorageAccountKey()));
444440
} else {
445-
client = ITestAbfsClient.setAbfsClientField(client, "tokenProvider",
446-
abfsConfig.getTokenProvider());
441+
ReflectionUtils.setFinalField(AbfsClient.class, client, "tokenProvider", abfsConfig.getTokenProvider());
447442
}
448443

449444
// override user agent
450445
String userAgent = "APN/1.0 Azure Blob FS/3.5.0-SNAPSHOT (PrivateBuild "
451446
+ "JavaJRE 1.8.0_252; Linux 5.3.0-59-generic/amd64; openssl-1.0; "
452447
+ "UNKNOWN/UNKNOWN) MSFT";
453-
client = ITestAbfsClient.setAbfsClientField(client, "userAgent", userAgent);
454-
455-
return client;
456-
}
448+
ReflectionUtils.setFinalField(AbfsClient.class, client, "userAgent", userAgent);
457449

458-
static AbfsClient setAbfsClientField(
459-
final AbfsClient client,
460-
final String fieldName,
461-
Object fieldObject) throws Exception {
462-
463-
Field field = AbfsClient.class.getDeclaredField(fieldName);
464-
field.setAccessible(true);
465-
Field modifiersField = Field.class.getDeclaredField("modifiers");
466-
modifiersField.setAccessible(true);
467-
modifiersField.setInt(field,
468-
field.getModifiers() & ~java.lang.reflect.Modifier.FINAL);
469-
field.set(client, fieldObject);
470450
return client;
471451
}
472452

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.hadoop.fs.permission.AclEntryScope;
4242
import org.apache.hadoop.fs.permission.AclEntryType;
4343
import org.apache.hadoop.fs.permission.FsAction;
44+
import org.apache.hadoop.test.ReflectionUtils;
4445
import org.apache.hadoop.util.Lists;
4546

4647
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
@@ -194,7 +195,7 @@ private void testRecursiveDeleteWithPaginationInternal(boolean isEmptyDir,
194195

195196
// Set the paginated enabled value and xMsVersion at spiedClient level.
196197
AbfsClient spiedClient = Mockito.spy(fs.getAbfsStore().getClient());
197-
ITestAbfsClient.setAbfsClientField(spiedClient, "xMsVersion", xMsVersion);
198+
ReflectionUtils.setFinalField(AbfsClient.class, spiedClient, "xMsVersion", xMsVersion);
198199
Mockito.doReturn(isPaginatedDeleteEnabled).when(spiedClient).getIsPaginatedDeleteEnabled();
199200

200201
AbfsRestOperation op = spiedClient.deletePath(

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java

Lines changed: 0 additions & 59 deletions
This file was deleted.

0 commit comments

Comments
 (0)