Skip to content

Commit 080fe14

Browse files
add test for marker
1 parent ddb87cc commit 080fe14

File tree

2 files changed

+155
-6
lines changed

2 files changed

+155
-6
lines changed

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1945,9 +1945,18 @@ private static String decodeMetadataAttribute(String encoded)
19451945
: URLDecoder.decode(encoded, StandardCharsets.UTF_8.name());
19461946
}
19471947

1948+
/**
1949+
* Checks if the listing of the specified path is non-empty.
1950+
*
1951+
* @param path The path to be listed.
1952+
* @param tracingContext The tracing context for tracking the operation.
1953+
* @return True if the listing is non-empty, False otherwise.
1954+
* @throws AzureBlobFileSystemException If an error occurs during the listing operation.
1955+
*/
19481956
private boolean isNonEmptyListing(String path,
19491957
TracingContext tracingContext) throws AzureBlobFileSystemException {
1950-
AbfsRestOperation listOp = listPath(path, false, 1, null, tracingContext, false);
1958+
AbfsRestOperation listOp = listPath(path, false, 1, null, tracingContext,
1959+
false);
19511960
return !isEmptyListResults(listOp.getResult());
19521961
}
19531962

@@ -1996,7 +2005,7 @@ public static String generateBlockListXml(List<String> blockIds) {
19962005
* @return true if the path is a directory and contains entries, false otherwise.
19972006
* @throws AzureBlobFileSystemException if the rest operation fails.
19982007
*/
1999-
private boolean checkIsDirectoryPath(String path,
2008+
public boolean checkIsDirectoryPath(String path,
20002009
TracingContext tracingContext)
20012010
throws AzureBlobFileSystemException {
20022011
// If listing returns non-empty results, return true else false.
@@ -2114,7 +2123,7 @@ HTTP_METHOD_PUT, createRequestUrl(path, EMPTY_STRING),
21142123
* @param tracingContext the tracing context.
21152124
* @throws AzureBlobFileSystemException if the creation of markers fails.
21162125
*/
2117-
private void createParentMarkersIfNeeded(String path,
2126+
public void createParentMarkersIfNeeded(String path,
21182127
AzureBlobFileSystemStore.Permissions permissions,
21192128
boolean isAppendBlob,
21202129
String eTag,
@@ -2197,6 +2206,21 @@ private AbfsRestOperation createFile(final String path,
21972206
isAppendBlob, eTag, contextEncryptionAdapter, tracingContext);
21982207
}
21992208

2209+
/**
2210+
* Retrieves the list of marker paths to be created for the specified path.
2211+
*
2212+
* @param path The path for which marker paths need to be created.
2213+
* @param tracingContext The tracing context for the operation.
2214+
* @return A list of paths that need to be created as markers.
2215+
* @throws AzureBlobFileSystemException If an error occurs while finding parent paths for marker creation.
2216+
*/
2217+
public List<Path> getMarkerPathsTobeCreated(final Path path,
2218+
final TracingContext tracingContext) throws AzureBlobFileSystemException {
2219+
ArrayList<Path> keysToCreateAsFolder = new ArrayList<>();
2220+
findParentPathsForMarkerCreation(path, tracingContext, keysToCreateAsFolder);
2221+
return keysToCreateAsFolder;
2222+
}
2223+
22002224
/**
22012225
* Creates marker blobs for the parent directories of the specified path.
22022226
*
@@ -2215,9 +2239,7 @@ private void createMarkers(final Path path,
22152239
final String eTag,
22162240
final ContextEncryptionAdapter contextEncryptionAdapter,
22172241
final TracingContext tracingContext) throws AzureBlobFileSystemException {
2218-
ArrayList<Path> keysToCreateAsFolder = new ArrayList<>();
2219-
findParentPathsForMarkerCreation(path, tracingContext,
2220-
keysToCreateAsFolder);
2242+
List<Path> keysToCreateAsFolder = getMarkerPathsTobeCreated(path, tracingContext);
22212243
for (Path pathToCreate : keysToCreateAsFolder) {
22222244
try {
22232245
createPathRestOp(pathToCreate.toUri().getPath(), false, false,

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.io.IOException;
2424
import java.lang.reflect.Field;
2525
import java.util.ArrayList;
26+
import java.util.Collections;
2627
import java.util.EnumSet;
2728
import java.util.List;
2829
import java.util.Random;
@@ -37,6 +38,8 @@
3738
import org.junit.Assume;
3839
import org.junit.Test;
3940
import org.mockito.Mockito;
41+
import org.mockito.invocation.InvocationOnMock;
42+
import org.mockito.stubbing.Answer;
4043

4144
import org.apache.hadoop.conf.Configuration;
4245
import org.apache.hadoop.fs.CreateFlag;
@@ -54,8 +57,10 @@
5457
import org.apache.hadoop.fs.azurebfs.services.AbfsBlobClient;
5558
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
5659
import org.apache.hadoop.fs.azurebfs.services.AbfsClientHandler;
60+
import org.apache.hadoop.fs.azurebfs.services.AbfsClientTestUtil;
5761
import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
5862
import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
63+
import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationType;
5964
import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
6065
import org.apache.hadoop.fs.azurebfs.services.RenameAtomicity;
6166
import org.apache.hadoop.fs.azurebfs.utils.DirectoryStateHelper;
@@ -85,6 +90,7 @@
8590
import static org.mockito.ArgumentMatchers.anyString;
8691
import static org.mockito.ArgumentMatchers.eq;
8792
import static org.mockito.ArgumentMatchers.nullable;
93+
import static org.mockito.Mockito.doAnswer;
8894
import static org.mockito.Mockito.doCallRealMethod;
8995
import static org.mockito.Mockito.doNothing;
9096
import static org.mockito.Mockito.doReturn;
@@ -729,6 +735,127 @@ public void testNegativeScenariosForCreateOverwriteDisabled()
729735
validateCreateFileException(AbfsRestOperationException.class, abfsStore);
730736
}
731737

738+
/**
739+
* Tests that the exception thrown during the creation of a marker is swallowed.
740+
* This test verifies that when an exception occurs during the creation of a marker,
741+
* it does not propagate and is handled internally and file creation still succeeds.
742+
*
743+
* @throws Throwable if an error occurs during the test execution
744+
*/
745+
@Test
746+
public void testCreateMarkerFailExceptionIsSwallowed()
747+
throws Throwable {
748+
749+
final AzureBlobFileSystem currentFs = getFileSystem();
750+
Configuration config = new Configuration(this.getRawConfiguration());
751+
config.set("fs.azure.enable.conditional.create.overwrite",
752+
Boolean.toString(true));
753+
754+
final AzureBlobFileSystem fs =
755+
(AzureBlobFileSystem) FileSystem.newInstance(currentFs.getUri(),
756+
config);
757+
758+
// Get mock AbfsClient with current config
759+
AbfsClient mockClient = Mockito.spy(fs.getAbfsClient());
760+
AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore());
761+
spiedStore.setClient(mockClient);
762+
763+
Assume.assumeTrue(mockClient instanceof AbfsBlobClient);
764+
AbfsClientHandler clientHandler = Mockito.mock(AbfsClientHandler.class);
765+
when(clientHandler.getIngressClient()).thenReturn(mockClient);
766+
when(clientHandler.getClient(Mockito.any())).thenReturn(mockClient);
767+
Path testFolder = new Path("/dir1");
768+
createAzCopyFolder(testFolder);
769+
770+
AzureBlobFileSystemStore abfsStore = fs.getAbfsStore();
771+
772+
ReflectionUtils.setFinalField(AzureBlobFileSystemStore.class, abfsStore,
773+
"clientHandler", clientHandler);
774+
ReflectionUtils.setFinalField(AzureBlobFileSystemStore.class, abfsStore,
775+
"client", mockClient);
776+
777+
AbfsRestOperation successOp = mock(
778+
AbfsRestOperation.class);
779+
AbfsHttpOperation http200Op = mock(
780+
AbfsHttpOperation.class);
781+
when(http200Op.getStatusCode()).thenReturn(HTTP_OK);
782+
when(successOp.getResult()).thenReturn(http200Op);
783+
784+
AbfsRestOperationException preConditionResponseEx
785+
= getMockAbfsRestOperationException(HTTP_PRECON_FAILED);
786+
787+
doCallRealMethod().when(mockClient)
788+
.conditionalCreateOverwriteFile(anyString(),
789+
Mockito.nullable(FileSystem.Statistics.class),
790+
Mockito.nullable(AzureBlobFileSystemStore.Permissions.class),
791+
anyBoolean(),
792+
Mockito.nullable(ContextEncryptionAdapter.class),
793+
Mockito.nullable(TracingContext.class));
794+
795+
doCallRealMethod().when((AbfsBlobClient) mockClient)
796+
.checkDirectoryAndCreateMarkersIfNeeded(anyString(),
797+
Mockito.nullable(AzureBlobFileSystemStore.Permissions.class),
798+
anyBoolean(),
799+
Mockito.nullable(String.class),
800+
Mockito.nullable(ContextEncryptionAdapter.class),
801+
Mockito.nullable(TracingContext.class));
802+
803+
doCallRealMethod().when((AbfsBlobClient) mockClient)
804+
.createParentMarkersIfNeeded(anyString(),
805+
Mockito.nullable(AzureBlobFileSystemStore.Permissions.class),
806+
anyBoolean(),
807+
Mockito.nullable(String.class),
808+
Mockito.nullable(ContextEncryptionAdapter.class),
809+
Mockito.nullable(TracingContext.class));
810+
811+
Mockito.doReturn(new ArrayList<>(Collections.singletonList(testFolder)))
812+
.when((AbfsBlobClient) mockClient)
813+
.getMarkerPathsTobeCreated(any(Path.class),
814+
Mockito.nullable(TracingContext.class));
815+
816+
doReturn(false).when((AbfsBlobClient) mockClient)
817+
.checkIsDirectoryPath(anyString(),
818+
Mockito.nullable(TracingContext.class));
819+
820+
// throw exception for first call of marker creation and return true for file creation
821+
doAnswer(new Answer<Void>() {
822+
private boolean firstCall = true;
823+
824+
@Override
825+
public Void answer(InvocationOnMock invocation) throws Throwable {
826+
if (firstCall) {
827+
firstCall = false;
828+
throw preConditionResponseEx;
829+
}
830+
return null;
831+
}
832+
}).doCallRealMethod()
833+
.when((AbfsBlobClient) mockClient)
834+
.createPathRestOp(anyString(), anyBoolean(), anyBoolean(),
835+
anyBoolean(), Mockito.nullable(String.class),
836+
Mockito.nullable(ContextEncryptionAdapter.class),
837+
Mockito.nullable(TracingContext.class));
838+
839+
AbfsClientTestUtil.hookOnRestOpsForTracingContextSingularity(mockClient);
840+
841+
doReturn(successOp) // Scn4: create overwrite=true fails with Http500
842+
.when((AbfsBlobClient) mockClient)
843+
.getPathStatus(any(String.class), any(TracingContext.class), nullable(
844+
ContextEncryptionAdapter.class), eq(false));
845+
846+
FsPermission permission = new FsPermission(FsAction.ALL, FsAction.ALL,
847+
FsAction.ALL);
848+
FsPermission umask = new FsPermission(FsAction.NONE, FsAction.NONE,
849+
FsAction.NONE);
850+
Path testPath = new Path("/dir1/testFile");
851+
abfsStore.createFile(testPath, null, true, permission, umask,
852+
getTestTracingContext(getFileSystem(), true));
853+
Assertions.assertThat(fs.exists(testPath))
854+
.describedAs("File not created when marker creation failed.")
855+
.isTrue();
856+
857+
}
858+
732859
private <E extends Throwable> void validateCreateFileException(final Class<E> exceptionClass, final AzureBlobFileSystemStore abfsStore)
733860
throws Exception {
734861
FsPermission permission = new FsPermission(FsAction.ALL, FsAction.ALL,

0 commit comments

Comments
 (0)