Skip to content

HADOOP-19448: [ABFS][FNSOverBlob] Optimizing the current create and mkdir flow #7353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Feb 7, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@
import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConcurrentWriteOperationDetectedException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.FileSystemOperationUnhandledException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidAbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidFileSystemPropertyException;
Expand Down Expand Up @@ -762,53 +761,8 @@ private AbfsRestOperation conditionalCreateOverwriteFile(final String relativePa
final TracingContext tracingContext) throws IOException {
AbfsRestOperation op;
AbfsClient createClient = getClientHandler().getIngressClient();
try {
// Trigger a create with overwrite=false first so that eTag fetch can be
// avoided for cases when no pre-existing file is present (major portion
// of create file traffic falls into the case of no pre-existing file).
op = createClient.createPath(relativePath, true, false, permissions,
isAppendBlob, null, contextEncryptionAdapter, tracingContext);

} catch (AbfsRestOperationException e) {
if (e.getStatusCode() == HttpURLConnection.HTTP_CONFLICT) {
// File pre-exists, fetch eTag
try {
op = getClient().getPathStatus(relativePath, false, tracingContext, null);
} catch (AbfsRestOperationException ex) {
if (ex.getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
// Is a parallel access case, as file which was found to be
// present went missing by this request.
throw new ConcurrentWriteOperationDetectedException(
"Parallel access to the create path detected. Failing request "
+ "to honor single writer semantics");
} else {
throw ex;
}
}

String eTag = extractEtagHeader(op.getResult());

try {
// overwrite only if eTag matches with the file properties fetched befpre
op = createClient.createPath(relativePath, true, true, permissions,
isAppendBlob, eTag, contextEncryptionAdapter, tracingContext);
} catch (AbfsRestOperationException ex) {
if (ex.getStatusCode() == HttpURLConnection.HTTP_PRECON_FAILED) {
// Is a parallel access case, as file with eTag was just queried
// and precondition failure can happen only when another file with
// different etag got created.
throw new ConcurrentWriteOperationDetectedException(
"Parallel access to the create path detected. Failing request "
+ "to honor single writer semantics");
} else {
throw ex;
}
}
} else {
throw e;
}
}

op = createClient.conditionalCreateOverwriteFile(relativePath, statistics,
permissions, isAppendBlob, contextEncryptionAdapter, tracingContext);
return op;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,29 @@

/**
* Thrown when a concurrent write operation is detected.
* This exception is used to indicate that parallel access to the create path
* has been detected, which violates the single writer semantics.
*/
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Evolving
public class ConcurrentWriteOperationDetectedException
extends AzureBlobFileSystemException {

private static final String ERROR_MESSAGE = "Parallel access to the create path detected. Failing request "
+ "to honor single writer semantics";

/**
* Constructs a new ConcurrentWriteOperationDetectedException with a default error message.
*/
public ConcurrentWriteOperationDetectedException() {
super(ERROR_MESSAGE);
}

/**
* Constructs a new ConcurrentWriteOperationDetectedException with the specified error message.
*
* @param message the detail message.
*/
public ConcurrentWriteOperationDetectedException(String message) {
super(message);
}
Expand Down
Loading