Skip to content

Commit 8532d9e

Browse files
HADOOP-18325: [ABFS] Fix metric related test failures due to missing config (#6847) (#7306)
Fixing test failures when the needed configs for metric collection are not set Contributed by: Anmol Asrani
1 parent e5ce6c0 commit 8532d9e

File tree

3 files changed

+46
-13
lines changed

3 files changed

+46
-13
lines changed

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import java.util.concurrent.TimeUnit;
4343
import java.util.concurrent.atomic.AtomicBoolean;
4444

45+
import org.apache.commons.lang3.StringUtils;
4546
import org.apache.hadoop.classification.VisibleForTesting;
4647
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
4748
import org.apache.hadoop.fs.azurebfs.constants.HttpOperationType;
@@ -247,21 +248,24 @@ private AbfsClient(final URL baseUrl,
247248
this.isMetricCollectionStopped = new AtomicBoolean(false);
248249
this.metricAnalysisPeriod = abfsConfiguration.getMetricAnalysisTimeout();
249250
this.metricIdlePeriod = abfsConfiguration.getMetricIdleTimeout();
250-
if (!metricFormat.toString().equals("")) {
251-
isMetricCollectionEnabled = true;
252-
abfsCounters.initializeMetrics(metricFormat);
251+
if (StringUtils.isNotEmpty(metricFormat.toString())) {
253252
String metricAccountName = abfsConfiguration.getMetricAccount();
254-
int dotIndex = metricAccountName.indexOf(AbfsHttpConstants.DOT);
255-
if (dotIndex <= 0) {
256-
throw new InvalidUriException(
257-
metricAccountName + " - account name is not fully qualified.");
258-
}
259253
String metricAccountKey = abfsConfiguration.getMetricAccountKey();
260-
try {
261-
metricSharedkeyCredentials = new SharedKeyCredentials(metricAccountName.substring(0, dotIndex),
262-
metricAccountKey);
263-
} catch (IllegalArgumentException e) {
264-
throw new IOException("Exception while initializing metric credentials " + e);
254+
if (StringUtils.isNotEmpty(metricAccountName) && StringUtils.isNotEmpty(metricAccountKey)) {
255+
isMetricCollectionEnabled = true;
256+
abfsCounters.initializeMetrics(metricFormat);
257+
int dotIndex = metricAccountName.indexOf(AbfsHttpConstants.DOT);
258+
if (dotIndex <= 0) {
259+
throw new InvalidUriException(
260+
metricAccountName + " - account name is not fully qualified.");
261+
}
262+
try {
263+
metricSharedkeyCredentials = new SharedKeyCredentials(
264+
metricAccountName.substring(0, dotIndex),
265+
metricAccountKey);
266+
} catch (IllegalArgumentException e) {
267+
throw new IOException("Exception while initializing metric credentials ", e);
268+
}
265269
}
266270
}
267271
if (isMetricCollectionEnabled) {

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@
2121
import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO;
2222
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE;
2323
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_WRITE_BUFFER_SIZE;
24+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY;
25+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME;
2426
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_FORMAT;
27+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI;
2528
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
2629
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_KB;
2730
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB;
@@ -30,6 +33,8 @@
3033
import org.apache.hadoop.conf.Configuration;
3134
import org.apache.hadoop.fs.FileSystem;
3235
import org.apache.hadoop.fs.azurebfs.utils.MetricFormat;
36+
37+
import org.junit.Assume;
3338
import org.junit.Test;
3439

3540
import java.io.IOException;
@@ -47,6 +52,20 @@
4752
public class ITestAbfsReadFooterMetrics extends AbstractAbfsScaleTest {
4853

4954
public ITestAbfsReadFooterMetrics() throws Exception {
55+
checkPrerequisites();
56+
}
57+
58+
private void checkPrerequisites(){
59+
checkIfConfigIsSet(FS_AZURE_METRIC_ACCOUNT_NAME);
60+
checkIfConfigIsSet(FS_AZURE_METRIC_ACCOUNT_KEY);
61+
checkIfConfigIsSet(FS_AZURE_METRIC_URI);
62+
}
63+
64+
private void checkIfConfigIsSet(String configKey){
65+
AbfsConfiguration conf = getConfiguration();
66+
String value = conf.get(configKey);
67+
Assume.assumeTrue(configKey + " config is mandatory for the test to run",
68+
value != null && value.trim().length() > 1);
5069
}
5170

5271
private static final String TEST_PATH = "/testfile";

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@
2424
import org.apache.hadoop.fs.azurebfs.utils.MetricFormat;
2525
import org.junit.Test;
2626
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_DELETE;
27+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY;
28+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME;
2729
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_FORMAT;
30+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI;
2831
import static org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationType.DeletePath;
2932
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
3033
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
@@ -39,6 +42,12 @@ public class TestAbfsRestOperation extends
3942
public TestAbfsRestOperation() throws Exception {
4043
}
4144

45+
private void checkPrerequisites() {
46+
assumeValidTestConfigPresent(getRawConfiguration(), FS_AZURE_METRIC_ACCOUNT_NAME);
47+
assumeValidTestConfigPresent(getRawConfiguration(), FS_AZURE_METRIC_ACCOUNT_KEY);
48+
assumeValidTestConfigPresent(getRawConfiguration(), FS_AZURE_METRIC_URI);
49+
}
50+
4251
/**
4352
* Test for backoff retry metrics.
4453
*
@@ -49,6 +58,7 @@ public TestAbfsRestOperation() throws Exception {
4958
*/
5059
@Test
5160
public void testBackoffRetryMetrics() throws Exception {
61+
checkPrerequisites();
5262
// Create an AzureBlobFileSystem instance.
5363
final Configuration configuration = getRawConfiguration();
5464
configuration.set(FS_AZURE_METRIC_FORMAT, String.valueOf(MetricFormat.INTERNAL_BACKOFF_METRIC_FORMAT));

0 commit comments

Comments
 (0)