Skip to content

Commit d02edaa

Browse files
JawedRakibJawedRakib
authored
Merge branch 'master' into big_query_fix
2 parents 6de198f + 51174a6 commit d02edaa

File tree

79 files changed

+2673
-445
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+2673
-445
lines changed

‎.github/workflows/run_release_tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
sudo apt update && sudo apt install -y tar gzip unzip wget less groff vim git python3 python3-pip unixodbc-dev curl jq;
1919
pip3 install boto3;
2020
- name: install node and npm
21-
uses: actions/setup-node@v4
21+
uses: actions/setup-node@v5
2222
with:
2323
node-version: 18
2424
- name: Set up JDK 11

‎athena-clickhouse/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
<dependency>
2323
<groupId>com.clickhouse</groupId>
2424
<artifactId>clickhouse-jdbc</artifactId>
25-
<version>0.9.1</version>
25+
<version>0.9.2</version>
2626
<classifier>all</classifier>
2727
</dependency>
2828
<dependency>

‎athena-cloudera-hive/META-INF/maven/org.apache.zookeeper/zookeeper-jute/pom.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
artifactId=zookeeper-jute
22
groupId=org.apache.zookeeper
3-
version=3.9.1
3+
version=3.9.4

‎athena-cloudera-hive/META-INF/maven/org.apache.zookeeper/zookeeper/pom.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
artifactId=zookeeper
22
groupId=org.apache.zookeeper
3-
version=3.9.1
3+
version=3.9.4

‎athena-cloudera-hive/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
<dependency>
3838
<groupId>org.apache.zookeeper</groupId>
3939
<artifactId>zookeeper</artifactId>
40-
<version>3.9.1</version>
40+
<version>3.9.4</version>
4141
</dependency>
4242
<dependency>
4343
<groupId>com.amazonaws</groupId>

‎athena-cloudera-hive/src/main/java/com/amazonaws/athena/connectors/cloudera/HiveJdbcConnectionFactory.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
package com.amazonaws.athena.connectors.cloudera;
2222

23+
import com.amazonaws.athena.connector.credentials.CredentialsConstants;
2324
import com.amazonaws.athena.connector.credentials.CredentialsProvider;
2425
import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionConfig;
2526
import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionInfo;
@@ -62,7 +63,7 @@ public Connection getConnection(final CredentialsProvider credentialsProvider)
6263
if (null != credentialsProvider) {
6364
Matcher secretMatcher = SECRET_NAME_PATTERN.matcher(databaseConnectionConfig.getJdbcConnectionString());
6465
final String secretReplacement = String.format("UID=%s;PWD=%s",
65-
credentialsProvider.getCredential().getUser(), credentialsProvider.getCredential().getPassword());
66+
credentialsProvider.getCredentialMap().get(CredentialsConstants.USER), credentialsProvider.getCredentialMap().get(CredentialsConstants.PASSWORD));
6667
derivedJdbcString = secretMatcher.replaceAll(Matcher.quoteReplacement(secretReplacement));
6768
}
6869
else {

‎athena-cloudera-impala/src/main/java/com/amazonaws/athena/connectors/cloudera/ImpalaJdbcConnectionFactory.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
package com.amazonaws.athena.connectors.cloudera;
2222

23+
import com.amazonaws.athena.connector.credentials.CredentialsConstants;
2324
import com.amazonaws.athena.connector.credentials.CredentialsProvider;
2425
import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionConfig;
2526
import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionInfo;
@@ -63,7 +64,7 @@ public Connection getConnection(final CredentialsProvider credentialsProvider)
6364
if (null != credentialsProvider) {
6465
Matcher secretMatcher = SECRET_NAME_PATTERN.matcher(databaseConnectionConfig.getJdbcConnectionString());
6566
final String secretReplacement = String.format("UID=%s;PWD=%s",
66-
credentialsProvider.getCredential().getUser(), credentialsProvider.getCredential().getPassword());
67+
credentialsProvider.getCredentialMap().get(CredentialsConstants.USER), credentialsProvider.getCredentialMap().get(CredentialsConstants.PASSWORD));
6768
derivedJdbcString = secretMatcher.replaceAll(Matcher.quoteReplacement(secretReplacement));
6869
}
6970
else {

‎athena-cloudwatch-metrics/src/main/java/com/amazonaws/athena/connectors/cloudwatch/metrics/MetricStatSerDe.java‎ renamed to ‎athena-cloudwatch-metrics/src/main/java/com/amazonaws/athena/connectors/cloudwatch/metrics/MetricDataQuerySerDe.java‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
* Licensed under the Apache License, Version 2.0 (the "License");
88
* you may not use this file except in compliance with the License.
99
* You may obtain a copy of the License at
10-
*
10+
*
1111
* http://www.apache.org/licenses/LICENSE-2.0
12-
*
12+
*
1313
* Unless required by applicable law or agreed to in writing, software
1414
* distributed under the License is distributed on an "AS IS" BASIS,
1515
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -22,50 +22,50 @@
2222
import com.fasterxml.jackson.core.JsonProcessingException;
2323
import com.fasterxml.jackson.databind.ObjectMapper;
2424
import com.fasterxml.jackson.databind.type.CollectionType;
25-
import software.amazon.awssdk.services.cloudwatch.model.MetricStat;
25+
import software.amazon.awssdk.services.cloudwatch.model.MetricDataQuery;
2626

2727
import java.io.IOException;
2828
import java.util.List;
2929
import java.util.stream.Collectors;
3030

3131
/**
32-
* Used to serialize and deserialize Cloudwatch Metrics MetricStat objects. This is used
32+
* Used to serialize and deserialize Cloudwatch Metrics MetricDataQuery objects. This is used
3333
* when creating and processing Splits.
3434
*/
35-
public class MetricStatSerDe
35+
public class MetricDataQuerySerDe
3636
{
37-
protected static final String SERIALIZED_METRIC_STATS_FIELD_NAME = "m";
37+
protected static final String SERIALIZED_METRIC_DATA_QUERIES_FIELD_NAME = "m";
3838
private static final ObjectMapper mapper = new ObjectMapper();
3939

40-
private MetricStatSerDe() {}
40+
private MetricDataQuerySerDe() {}
4141

4242
/**
43-
* Serializes the provided List of MetricStats.
43+
* Serializes the provided List of MetricDataQueries.
4444
*
45-
* @param metricStats The list of MetricStats to serialize.
46-
* @return A String containing the serialized list of MetricStats.
45+
* @param metricDataQueries The list of MetricDataQueries to serialize.
46+
* @return A String containing the serialized list of MetricDataQueries.
4747
*/
48-
public static String serialize(List<MetricStat> metricStats)
48+
public static String serialize(List<MetricDataQuery> metricDataQueries)
4949
{
5050
try {
51-
return mapper.writeValueAsString(metricStats.stream().map(stat -> stat.toBuilder()).collect(Collectors.toList()));
51+
return mapper.writeValueAsString(metricDataQueries.stream().map(query -> query.toBuilder()).collect(Collectors.toList()));
5252
}
5353
catch (JsonProcessingException ex) {
5454
throw new RuntimeException(ex);
5555
}
5656
}
5757

5858
/**
59-
* Deserializes the provided String into a List of MetricStats.
59+
* Deserializes the provided String into a List of MetricDataQueries.
6060
*
61-
* @param serializedMetricStats A serialized list of MetricStats.
62-
* @return The List of MetricStats represented by the serialized string.
61+
* @param serializedMetricDataQueries A serialized list of MetricDataQueries.
62+
* @return The List of MetricDataQueries represented by the serialized string.
6363
*/
64-
public static List<MetricStat> deserialize(String serializedMetricStats)
64+
public static List<MetricDataQuery> deserialize(String serializedMetricDataQueries)
6565
{
6666
try {
67-
CollectionType metricStatBuilderCollection = mapper.getTypeFactory().constructCollectionType(List.class, MetricStat.serializableBuilderClass());
68-
return ((List<MetricStat.Builder>) mapper.readValue(serializedMetricStats, metricStatBuilderCollection)).stream().map(stat -> stat.build()).collect(Collectors.toList());
67+
CollectionType metricDataQueryBuilderCollection = mapper.getTypeFactory().constructCollectionType(List.class, MetricDataQuery.serializableBuilderClass());
68+
return ((List<MetricDataQuery.Builder>) mapper.readValue(serializedMetricDataQueries, metricDataQueryBuilderCollection)).stream().map(query -> query.build()).collect(Collectors.toList());
6969
}
7070
catch (IOException ex) {
7171
throw new RuntimeException(ex);

‎athena-cloudwatch-metrics/src/main/java/com/amazonaws/athena/connectors/cloudwatch/metrics/MetricUtils.java‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,7 @@
3434
import software.amazon.awssdk.services.cloudwatch.model.ListMetricsRequest;
3535
import software.amazon.awssdk.services.cloudwatch.model.Metric;
3636
import software.amazon.awssdk.services.cloudwatch.model.MetricDataQuery;
37-
import software.amazon.awssdk.services.cloudwatch.model.MetricStat;
3837

39-
import java.util.ArrayList;
4038
import java.util.Collections;
4139
import java.util.Date;
4240
import java.util.List;
@@ -151,16 +149,10 @@ protected static void pushDownPredicate(Constraints constraints, ListMetricsRequ
151149
protected static GetMetricDataRequest makeGetMetricDataRequest(ReadRecordsRequest readRecordsRequest)
152150
{
153151
Split split = readRecordsRequest.getSplit();
154-
String serializedMetricStats = split.getProperty(MetricStatSerDe.SERIALIZED_METRIC_STATS_FIELD_NAME);
155-
List<MetricStat> metricStats = MetricStatSerDe.deserialize(serializedMetricStats);
152+
String serializedMetricDataQueries = split.getProperty(MetricDataQuerySerDe.SERIALIZED_METRIC_DATA_QUERIES_FIELD_NAME);
153+
List<MetricDataQuery> metricDataQueries = MetricDataQuerySerDe.deserialize(serializedMetricDataQueries);
156154
GetMetricDataRequest.Builder dataRequestBuilder = GetMetricDataRequest.builder();
157155

158-
List<MetricDataQuery> metricDataQueries = new ArrayList<>();
159-
int metricId = 1;
160-
for (MetricStat nextMetricStat : metricStats) {
161-
metricDataQueries.add(MetricDataQuery.builder().metricStat(nextMetricStat).id("m" + metricId++).build());
162-
}
163-
164156
dataRequestBuilder.metricDataQueries(metricDataQueries);
165157

166158
ValueSet timeConstraint = readRecordsRequest.getConstraints().getSummary().get(TIMESTAMP_FIELD);

‎athena-cloudwatch-metrics/src/main/java/com/amazonaws/athena/connectors/cloudwatch/metrics/MetricsMetadataHandler.java‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import software.amazon.awssdk.services.cloudwatch.model.ListMetricsRequest;
5252
import software.amazon.awssdk.services.cloudwatch.model.ListMetricsResponse;
5353
import software.amazon.awssdk.services.cloudwatch.model.Metric;
54+
import software.amazon.awssdk.services.cloudwatch.model.MetricDataQuery;
5455
import software.amazon.awssdk.services.cloudwatch.model.MetricStat;
5556
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
5657
import software.amazon.awssdk.utils.CollectionUtils;
@@ -243,19 +244,29 @@ public GetSplitsResponse doGetSplits(BlockAllocator blockAllocator, GetSplitsReq
243244
ListMetricsRequest listMetricsRequest = listMetricsRequestBuilder.build();
244245
ListMetricsResponse result = invoker.invoke(() -> metrics.listMetrics(listMetricsRequest));
245246

246-
List<MetricStat> metricStats = new ArrayList<>(100);
247-
for (Metric nextMetric : result.metrics()) {
247+
List<MetricDataQuery> metricDataQueries = new ArrayList<>(100);
248+
List<Metric> metrics = result.metrics();
249+
List<String> accounts = result.owningAccounts();
250+
251+
// There is a 1:1 mapping between each metric that is returned and the ID of the owning account.
252+
for (int i = 0; i < metrics.size(); i++) {
253+
Metric metric = metrics.get(i);
248254
for (String nextStatistic : STATISTICS) {
249-
if (MetricUtils.applyMetricConstraints(constraintEvaluator, nextMetric, nextStatistic)) {
250-
metricStats.add(MetricStat.builder()
251-
.metric(Metric.builder()
252-
.namespace(nextMetric.namespace())
253-
.metricName(nextMetric.metricName())
254-
.dimensions(nextMetric.dimensions())
255-
.build())
256-
.period(Integer.valueOf(period))
257-
.stat(nextStatistic)
258-
.build());
255+
if (MetricUtils.applyMetricConstraints(constraintEvaluator, metric, nextStatistic)) {
256+
metricDataQueries.add(MetricDataQuery.builder()
257+
.metricStat(MetricStat.builder()
258+
.metric(Metric.builder()
259+
.namespace(metric.namespace())
260+
.metricName(metric.metricName())
261+
.dimensions(metric.dimensions())
262+
.build())
263+
.period(Integer.valueOf(period))
264+
.stat(nextStatistic)
265+
.build())
266+
.id("m" + (i + 1))
267+
.accountId(accounts.isEmpty() ? null : accounts.get(i))
268+
.build()
269+
);
259270
}
260271
}
261272
}
@@ -266,16 +277,16 @@ public GetSplitsResponse doGetSplits(BlockAllocator blockAllocator, GetSplitsReq
266277
continuationToken = result.nextToken();
267278
}
268279

269-
if (CollectionUtils.isNullOrEmpty(metricStats)) {
270-
logger.info("No metric stats present after filtering predicates.");
280+
if (CollectionUtils.isNullOrEmpty(metricDataQueries)) {
281+
logger.info("No metric data queries present after filtering predicates.");
271282
return new GetSplitsResponse(getSplitsRequest.getCatalogName(), splits, continuationToken);
272283
}
273284

274-
List<List<MetricStat>> partitions = Lists.partition(metricStats, calculateSplitSize(metricStats.size()));
275-
for (List<MetricStat> partition : partitions) {
276-
String serializedMetricStats = MetricStatSerDe.serialize(partition);
285+
List<List<MetricDataQuery>> partitions = Lists.partition(metricDataQueries, calculateSplitSize(metricDataQueries.size()));
286+
for (List<MetricDataQuery> partition : partitions) {
287+
String serializedMetricDataQueries = MetricDataQuerySerDe.serialize(partition);
277288
splits.add(Split.newBuilder(makeSpillLocation(getSplitsRequest), makeEncryptionKey())
278-
.add(MetricStatSerDe.SERIALIZED_METRIC_STATS_FIELD_NAME, serializedMetricStats)
289+
.add(MetricDataQuerySerDe.SERIALIZED_METRIC_DATA_QUERIES_FIELD_NAME, serializedMetricDataQueries)
279290
.build());
280291
}
281292

0 commit comments

Comments
 (0)