Skip to content

Commit 68dbb3c

Browse files
anujmodi2021anujmodi2021
committed
HADOOP-19443: [ABFS][FnsOverBlob] Updating hadoop-azure Test Scripts and Adding Tests For Metadata APIs (apache#7344)
Contributed by Anuj Modi Reviewed by Anmol Asrani, Sneha Vijayarajan, Manish Bhatt
1 parent 4a217aa commit 68dbb3c

17 files changed

+558
-110
lines changed

‎hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh

Lines changed: 102 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -32,36 +32,87 @@ processCount=8
3232

3333
## SECTION: TEST COMBINATION METHODS
3434

35-
runHNSOAuthTest()
35+
runHNSOAuthDFSTest()
3636
{
3737
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
3838
PROPERTIES=("fs.azure.account.auth.type")
3939
VALUES=("OAuth")
40-
triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
40+
triggerRun "HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
4141
}
4242

43-
runHNSSharedKeyTest()
43+
runHNSSharedKeyDFSTest()
4444
{
4545
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
4646
PROPERTIES=("fs.azure.account.auth.type")
4747
VALUES=("SharedKey")
48-
triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
48+
triggerRun "HNS-SharedKey-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
4949
}
5050

51-
runNonHNSSharedKeyTest()
51+
runNonHNSSharedKeyDFSTest()
5252
{
5353
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
5454
PROPERTIES=("fs.azure.account.auth.type")
5555
VALUES=("SharedKey")
56-
triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
56+
triggerRun "NonHNS-SharedKey-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
5757
}
5858

59-
runAppendBlobHNSOAuthTest()
59+
runAppendBlobHNSOAuthDFSTest()
6060
{
6161
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
6262
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
6363
VALUES=("OAuth" "true")
64-
triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
64+
triggerRun "AppendBlob-HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
65+
}
66+
67+
runNonHNSSharedKeyBlobTest()
68+
{
69+
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
70+
fnsBlobConfigFileCheck "$accountName"
71+
PROPERTIES=("fs.azure.account.auth.type")
72+
VALUES=("SharedKey")
73+
triggerRun "NonHNS-SharedKey-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers"
74+
}
75+
76+
runNonHNSOAuthDFSTest()
77+
{
78+
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
79+
PROPERTIES=("fs.azure.account.auth.type")
80+
VALUES=("OAuth")
81+
triggerRun "NonHNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
82+
}
83+
84+
runNonHNSOAuthBlobTest()
85+
{
86+
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
87+
fnsBlobConfigFileCheck "$accountName"
88+
PROPERTIES=("fs.azure.account.auth.type")
89+
VALUES=("OAuth")
90+
triggerRun "NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers"
91+
}
92+
93+
runAppendBlobNonHNSOAuthBlobTest()
94+
{
95+
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
96+
fnsBlobConfigFileCheck "$accountName"
97+
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
98+
VALUES=("OAuth" "true")
99+
triggerRun "AppendBlob-NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers"
100+
}
101+
102+
runHNSOAuthDFSIngressBlobTest()
103+
{
104+
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
105+
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type")
106+
VALUES=("OAuth" "blob")
107+
triggerRun "HNS-Oauth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
108+
}
109+
110+
runNonHNSOAuthDFSIngressBlobTest()
111+
{
112+
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
113+
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type")
114+
VALUES=("OAuth" "blob")
115+
triggerRun "NonHNS-OAuth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
65116
}
66117

67118
runTest=false
@@ -130,23 +181,47 @@ done
130181

131182
echo ' '
132183
echo 'Set the active test combination to run the action:'
133-
select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
184+
select combo in HNS-OAuth-DFS HNS-SharedKey-DFS NonHNS-SharedKey-DFS AppendBlob-HNS-OAuth-DFS NonHNS-SharedKey-Blob NonHNS-OAuth-DFS NonHNS-OAuth-Blob AppendBlob-NonHNS-OAuth-Blob HNS-Oauth-DFS-IngressBlob NonHNS-Oauth-DFS-IngressBlob AllCombinationsTestRun Quit
134185
do
135186
case $combo in
136-
HNS-OAuth)
137-
runHNSOAuthTest
187+
HNS-OAuth-DFS)
188+
runHNSOAuthDFSTest
189+
break
190+
;;
191+
HNS-SharedKey-DFS)
192+
runHNSSharedKeyDFSTest
193+
break
194+
;;
195+
NonHNS-SharedKey-DFS)
196+
runNonHNSSharedKeyDFSTest
197+
break
198+
;;
199+
AppendBlob-HNS-OAuth-DFS)
200+
runAppendBlobHNSOAuthDFSTest
201+
break
202+
;;
203+
NonHNS-SharedKey-Blob)
204+
runNonHNSSharedKeyBlobTest
205+
break
206+
;;
207+
NonHNS-OAuth-DFS)
208+
runNonHNSOAuthDFSTest
209+
break
210+
;;
211+
NonHNS-OAuth-Blob)
212+
runNonHNSOAuthBlobTest
138213
break
139214
;;
140-
HNS-SharedKey)
141-
runHNSSharedKeyTest
215+
AppendBlob-NonHNS-OAuth-Blob)
216+
runAppendBlobNonHNSOAuthBlobTest
142217
break
143218
;;
144-
nonHNS-SharedKey)
145-
runNonHNSSharedKeyTest
219+
HNS-Oauth-DFS-IngressBlob)
220+
runHNSOAuthDFSIngressBlobTest
146221
break
147222
;;
148-
AppendBlob-HNS-OAuth)
149-
runAppendBlobHNSOAuthTest
223+
NonHNS-Oauth-DFS-IngressBlob)
224+
runNonHNSOAuthDFSIngressBlobTest
150225
break
151226
;;
152227
AllCombinationsTestRun)
@@ -155,10 +230,16 @@ do
155230
logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
156231
break
157232
fi
158-
runHNSOAuthTest
159-
runHNSSharedKeyTest
160-
runNonHNSSharedKeyTest
161-
runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
233+
runHNSOAuthDFSTest
234+
runHNSSharedKeyDFSTest
235+
runNonHNSSharedKeyDFSTest
236+
runAppendBlobHNSOAuthDFSTest
237+
runNonHNSSharedKeyBlobTest
238+
runNonHNSOAuthDFSTest
239+
runNonHNSOAuthBlobTest
240+
runAppendBlobNonHNSOAuthBlobTest
241+
runHNSOAuthDFSIngressBlobTest
242+
runNonHNSOAuthDFSIngressBlobTest
162243
break
163244
;;
164245
Quit)

‎hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,25 @@ ENDTIME=$(date +%s)
4545
outputFormatOn="\033[0;95m"
4646
outputFormatOff="\033[0m"
4747

48+
# Function to check if the blob config file exists and create one if it doesn't.
49+
fnsBlobConfigFileCheck() {
50+
baseFileName=$1
51+
targetWord=".dfs."
52+
replacementWord=".blob."
53+
accountSettingsDir="src/test/resources/accountSettings/"
54+
accountConfigFileSuffix="_settings.xml"
55+
sourceFilePath="${accountSettingsDir}${baseFileName}${accountConfigFileSuffix}"
56+
targetFilePath="${accountSettingsDir}${baseFileName}_blob${accountConfigFileSuffix}"
57+
58+
if [ ! -f "$targetFilePath" ]; then
59+
cp "$sourceFilePath" "$targetFilePath"
60+
sed -i "s/$targetWord/$replacementWord/g" "$targetFilePath"
61+
echo "File created and word replaced."
62+
else
63+
echo "File already exists."
64+
fi
65+
}
66+
4867
triggerRun()
4968
{
5069
echo ' '

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -543,11 +543,6 @@ public boolean isDfsToBlobFallbackEnabled() {
543543
*/
544544
public void validateConfiguredServiceType(boolean isHNSEnabled)
545545
throws InvalidConfigurationValueException {
546-
// TODO: [FnsOverBlob][HADOOP-19179] Remove this check when FNS over Blob is ready.
547-
if (getFsConfiguredServiceType() == AbfsServiceType.BLOB) {
548-
throw new InvalidConfigurationValueException(FS_DEFAULT_NAME_KEY,
549-
"Blob Endpoint Support not yet available");
550-
}
551546
if (isHNSEnabled && getConfiguredServiceTypeForFNSAccounts() == AbfsServiceType.BLOB) {
552547
throw new InvalidConfigurationValueException(
553548
FS_AZURE_FNS_ACCOUNT_SERVICE_TYPE, "Service Type Cannot be BLOB for HNS Account");

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ public void testNonExistingPath() throws Exception {
225225

226226
private void verifyIteratorResultContent(FileStatus fileStatus,
227227
List<String> fileNames) {
228+
assertPathDns(fileStatus.getPath());
228229
String pathStr = fileStatus.getPath().toString();
229230
Assert.assertTrue(
230231
String.format("Could not remove path %s from filenames %s", pathStr,

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -340,9 +340,7 @@ public void testCreateAppendBlobOverBlobEndpointAppendOverDfs()
340340
@Test
341341
public void testCreateAppendBlobOverDfsEndpointAppendOverBlob()
342342
throws IOException, NoSuchFieldException, IllegalAccessException {
343-
Assume.assumeTrue(
344-
"FNS does not support append blob creation for DFS endpoint",
345-
getIsNamespaceEnabled(getFileSystem()));
343+
assumeHnsEnabled("FNS does not support append blob creation for DFS endpoint");
346344
final AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
347345
AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
348346
Mockito.doReturn(true).when(store).isAppendBlobKey(anyString());
@@ -746,7 +744,7 @@ public void testIntermittentAppendFailureToBeReported() throws Exception {
746744
Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
747745
try (AzureBlobFileSystem fs = Mockito.spy(
748746
(AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()))) {
749-
Assume.assumeTrue(!getIsNamespaceEnabled(fs));
747+
assumeHnsDisabled();
750748
AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
751749
assumeBlobServiceType();
752750

@@ -951,8 +949,7 @@ public void testFlushSuccessWithConnectionResetOnResponseValidMd5() throws Excep
951949
// Create a spy of AzureBlobFileSystem
952950
try (AzureBlobFileSystem fs = Mockito.spy(
953951
(AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()))) {
954-
Assume.assumeTrue(!getIsNamespaceEnabled(fs));
955-
952+
assumeHnsDisabled();
956953
// Create a spy of AzureBlobFileSystemStore
957954
AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
958955
assumeBlobServiceType();
@@ -1049,7 +1046,7 @@ public void testFlushSuccessWithConnectionResetOnResponseInvalidMd5() throws Exc
10491046
// Create a spy of AzureBlobFileSystem
10501047
try (AzureBlobFileSystem fs = Mockito.spy(
10511048
(AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()))) {
1052-
Assume.assumeTrue(!getIsNamespaceEnabled(fs));
1049+
assumeHnsDisabled();
10531050

10541051
// Create a spy of AzureBlobFileSystemStore
10551052
AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());

0 commit comments

Comments
 (0)