From 63d443bd7416a781afb18c5fede09127fc67d2ff Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 20 Dec 2016 16:11:29 -0800 Subject: [PATCH 1/2] BigQuery: add samples for query parameters. Samples for: - Named parameters with scalar values. - Named parameters with array values. - Named parameters with timestamp values. --- bigquery/cloud-client/pom.xml | 5 + .../bigquery/QueryParametersSample.java | 275 ++++++++++++++++++ .../bigquery/QueryParametersSampleIT.java | 71 +++++ 3 files changed, 351 insertions(+) create mode 100644 bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java create mode 100644 bigquery/cloud-client/src/test/java/com/example/bigquery/QueryParametersSampleIT.java diff --git a/bigquery/cloud-client/pom.xml b/bigquery/cloud-client/pom.xml index f6bb3cace9d..4a7badc1757 100644 --- a/bigquery/cloud-client/pom.xml +++ b/bigquery/cloud-client/pom.xml @@ -39,6 +39,11 @@ google-cloud-bigquery 0.8.0-beta + + joda-time + joda-time + 2.9.7 + diff --git a/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java b/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java new file mode 100644 index 00000000000..d9f90ff1da5 --- /dev/null +++ b/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java @@ -0,0 +1,275 @@ +/* + Copyright 2016 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.example.bigquery; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.QueryParameterValue; +import com.google.cloud.bigquery.QueryRequest; +import com.google.cloud.bigquery.QueryResponse; +import com.google.cloud.bigquery.QueryResult; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; + +/** + * A sample that demonstrates use of query parameters. + */ +public class QueryParametersSample { + private static final int ERROR_CODE = 1; + + private static void printUsage() { + System.err.println("Usage:"); + System.err.printf( + "mvn exec:java -Dexec.mainClass=%s -Dexec.args=%s\n", + QueryParametersSample.class.getCanonicalName(), + "sample"); + System.err.println("sample values: named|array|struct|timestamp"); + System.err.println("Usage for sample=named:"); + System.err.println("\tnamed corpus minWordCount"); + System.err.println("Usage for sample=array:"); + System.err.println("\tarray gender states..."); + System.err.println("\tgender=M|F"); + System.err.println("\tstates=Upper-case 2-letter code for U.S. state, e.g. CA."); + } + + /** + * Prompts the user for the required parameters to perform a query. + */ + public static void main(final String[] args) throws IOException, InterruptedException { + if (args.length < 1) { + System.err.println("Expected first argument 'sample'"); + printUsage(); + System.exit(ERROR_CODE); + } + String sample = args[0]; + + switch (sample) { + case "named": + if (args.length != 3) { + System.err.println("Unexpected number of arguments for named query sample."); + printUsage(); + System.exit(ERROR_CODE); + } + runNamed(args[1], Long.parseLong(args[2])); + break; + case "array": + if (args.length < 2) { + System.err.println("Unexpected number of arguments for array query sample."); + printUsage(); + System.exit(ERROR_CODE); + } + String gender = args[1]; + ArrayList states = new ArrayList<>(); + for (int i = 2; i < args.length; i++) { + states.add(args[i]); + } + runArray(gender, states); + break; + case "timestamp": + if (args.length != 1) { + System.err.println("Unexpected number of arguments for timestamp query sample."); + printUsage(); + System.exit(ERROR_CODE); + } + runTimestamp(); + break; + default: + System.err.println("Got bad value for sample"); + printUsage(); + System.exit(ERROR_CODE); + } + } + + /** + * Query the Shakespeare dataset for words with count at least {@code minWordCount} in the corpus + * {@code corpus}. + */ + // [START bigquery_query_params] + private static void runNamed(final String corpus, final long minWordCount) + throws InterruptedException { + BigQuery bigquery = + new BigQueryOptions.DefaultBigqueryFactory().create(BigQueryOptions.getDefaultInstance()); + + String queryString = "SELECT word, word_count\n" + + "FROM `bigquery-public-data.samples.shakespeare`\n" + + "WHERE corpus = @corpus\n" + + "AND word_count >= @min_word_count\n" + + "ORDER BY word_count DESC"; + QueryRequest queryRequest = + QueryRequest.newBuilder(queryString) + .addNamedParameter("corpus", QueryParameterValue.string(corpus)) + .addNamedParameter("min_word_count", QueryParameterValue.int64(minWordCount)) + // Standard SQL syntax is required for parameterized queries. + // See: https://cloud.google.com/bigquery/sql-reference/ + .setUseLegacySql(false) + .build(); + + // Execute the query. + QueryResponse response = bigquery.query(queryRequest); + + // Wait for the job to finish (if the query takes more than 10 seconds to complete). + while (!response.jobCompleted()) { + Thread.sleep(1000); + response = bigquery.getQueryResults(response.getJobId()); + } + + if (response.hasErrors()) { + throw new RuntimeException( + response + .getExecutionErrors() + .stream() + .map(err -> err.getMessage()) + .collect(Collectors.joining("\n"))); + } + + QueryResult result = response.getResult(); + Iterator> iter = result.iterateAll(); + + while (iter.hasNext()) { + List row = iter.next(); + System.out.printf( + "%s: %d\n", + row.get(0).getStringValue(), + row.get(1).getLongValue()); + } + } + // [END bigquery_query_params] + + /** + * Query the baby names database to find the most popular names for a gender in a list of states. + */ + // [START bigquery_query_params_arrays] + private static void runArray(String gender, List states) + throws InterruptedException { + BigQuery bigquery = + new BigQueryOptions.DefaultBigqueryFactory().create(BigQueryOptions.getDefaultInstance()); + + String queryString = "SELECT name, sum(number) as count\n" + + "FROM `bigquery-public-data.usa_names.usa_1910_2013`\n" + + "WHERE gender = @gender\n" + + "AND state IN UNNEST(@states)\n" + + "GROUP BY name\n" + + "ORDER BY count DESC\n" + + "LIMIT 10;"; + QueryRequest queryRequest = + QueryRequest.newBuilder(queryString) + .addNamedParameter("gender", QueryParameterValue.string(gender)) + .addNamedParameter( + "states", + QueryParameterValue.array( + states.toArray(new String[]{}), + String.class)) + // Standard SQL syntax is required for parameterized queries. + // See: https://cloud.google.com/bigquery/sql-reference/ + .setUseLegacySql(false) + .build(); + + // Execute the query. + QueryResponse response = bigquery.query(queryRequest); + + // Wait for the job to finish (if the query takes more than 10 seconds to complete). + while (!response.jobCompleted()) { + Thread.sleep(1000); + response = bigquery.getQueryResults(response.getJobId()); + } + + if (response.hasErrors()) { + throw new RuntimeException( + response + .getExecutionErrors() + .stream() + .map(err -> err.getMessage()) + .collect(Collectors.joining("\n"))); + } + + QueryResult result = response.getResult(); + Iterator> iter = result.iterateAll(); + + while (iter.hasNext()) { + List row = iter.next(); + System.out.printf("%s: %d\n", row.get(0).getStringValue(), row.get(1).getLongValue()); + } + } + // [END bigquery_query_params_arrays] + + // [START bigquery_query_params_timestamps] + private static void runTimestamp() throws InterruptedException { + BigQuery bigquery = + new BigQueryOptions.DefaultBigqueryFactory().create(BigQueryOptions.getDefaultInstance()); + + // Timestamps are expected to be in the format YYYY-MM-DD HH:MM:SS.DDDDDD time_zone + // See: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#timestamp-type + DateTime timestamp = new DateTime(2016, 12, 7, 8, 0, 0, DateTimeZone.UTC); + + String queryString = "SELECT TIMESTAMP_ADD(@ts_value, INTERVAL 1 HOUR);"; + QueryRequest queryRequest = + QueryRequest.newBuilder(queryString) + .addNamedParameter( + "ts_value", + QueryParameterValue.timestamp( + // Timestamp takes microseconds since 1970-01-01T00:00:00 UTC + timestamp.getMillis() * 1000)) + // Standard SQL syntax is required for parameterized queries. + // See: https://cloud.google.com/bigquery/sql-reference/ + .setUseLegacySql(false) + .build(); + + // Execute the query. + QueryResponse response = bigquery.query(queryRequest); + + // Wait for the job to finish (if the query takes more than 10 seconds to complete). + while (!response.jobCompleted()) { + Thread.sleep(1000); + response = bigquery.getQueryResults(response.getJobId()); + } + + if (response.hasErrors()) { + throw new RuntimeException( + response + .getExecutionErrors() + .stream() + .map(err -> err.getMessage()) + .collect(Collectors.joining("\n"))); + } + + QueryResult result = response.getResult(); + Iterator> iter = result.iterateAll(); + + DateTimeFormatter formatter = ISODateTimeFormat.dateTimeNoMillis().withZoneUTC(); + while (iter.hasNext()) { + List row = iter.next(); + System.out.printf( + "%s\n", + formatter.print( + new DateTime( + // Timestamp values are returned in microseconds since 1970-01-01T00:00:00 UTC, + // but org.joda.time.DateTime constructor accepts times in milliseconds. + row.get(0).getTimestampValue() / 1000, + DateTimeZone.UTC))); + } + } + // [END bigquery_query_params_timestamps] +} diff --git a/bigquery/cloud-client/src/test/java/com/example/bigquery/QueryParametersSampleIT.java b/bigquery/cloud-client/src/test/java/com/example/bigquery/QueryParametersSampleIT.java new file mode 100644 index 00000000000..4577850d872 --- /dev/null +++ b/bigquery/cloud-client/src/test/java/com/example/bigquery/QueryParametersSampleIT.java @@ -0,0 +1,71 @@ +/* + Copyright 2016 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +/** + * Tests for simple app sample. + */ +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class QueryParametersSampleIT { + private ByteArrayOutputStream bout; + private PrintStream out; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testNamedSample() throws Exception { + QueryParametersSample.main(new String[]{"named", "romeoandjuliet", "100"}); + String got = bout.toString(); + assertThat(got).contains("love"); + } + + @Test + public void testArraySample() throws Exception { + QueryParametersSample.main(new String[]{"array", "M", "WA", "WI", "WV", "WY"}); + String got = bout.toString(); + assertThat(got).contains("James"); + } + + @Test + public void testTimestampSample() throws Exception { + QueryParametersSample.main(new String[]{"timestamp"}); + String got = bout.toString(); + assertThat(got).contains("2016-12-07T09:00:00Z"); + } +} From 682d7652d74f0accce42cf6ad72aa1f7355dc9be Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 21 Dec 2016 11:39:12 -0800 Subject: [PATCH 2/2] Clarify usage examples. Also, switch runArray from taking a list to taking an array. --- .../bigquery/QueryParametersSample.java | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java b/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java index d9f90ff1da5..b2ee9362b75 100644 --- a/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java +++ b/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java @@ -29,7 +29,7 @@ import org.joda.time.format.ISODateTimeFormat; import java.io.IOException; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; @@ -43,16 +43,32 @@ public class QueryParametersSample { private static void printUsage() { System.err.println("Usage:"); System.err.printf( - "mvn exec:java -Dexec.mainClass=%s -Dexec.args=%s\n", + "\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n", QueryParametersSample.class.getCanonicalName(), - "sample"); - System.err.println("sample values: named|array|struct|timestamp"); - System.err.println("Usage for sample=named:"); - System.err.println("\tnamed corpus minWordCount"); + "${sample}"); + System.err.println(); + System.err.println("${sample} can be one of: named, array, timestamp"); + System.err.println(); + System.err.println("Usage for ${sample}=named:"); + System.err.printf( + "\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n", + QueryParametersSample.class.getCanonicalName(), + "named ${corpus} ${minWordCount}"); + System.err.println(); System.err.println("Usage for sample=array:"); - System.err.println("\tarray gender states..."); - System.err.println("\tgender=M|F"); - System.err.println("\tstates=Upper-case 2-letter code for U.S. state, e.g. CA."); + System.err.printf( + "\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n", + QueryParametersSample.class.getCanonicalName(), + "array ${gender} ${states...}"); + System.err.println(); + System.err.println("\twhere ${gender} can be on of: M, F"); + System.err.println( + "\tand ${states} is any upper-case 2-letter code for U.S. a state, e.g. CA."); + System.err.println(); + System.err.printf( + "\t\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n", + QueryParametersSample.class.getCanonicalName(), + "array F MD WA"); } /** @@ -82,10 +98,7 @@ public static void main(final String[] args) throws IOException, InterruptedExce System.exit(ERROR_CODE); } String gender = args[1]; - ArrayList states = new ArrayList<>(); - for (int i = 2; i < args.length; i++) { - states.add(args[i]); - } + String[] states = Arrays.copyOfRange(args, 2, args.length); runArray(gender, states); break; case "timestamp": @@ -162,7 +175,7 @@ private static void runNamed(final String corpus, final long minWordCount) * Query the baby names database to find the most popular names for a gender in a list of states. */ // [START bigquery_query_params_arrays] - private static void runArray(String gender, List states) + private static void runArray(String gender, String[] states) throws InterruptedException { BigQuery bigquery = new BigQueryOptions.DefaultBigqueryFactory().create(BigQueryOptions.getDefaultInstance()); @@ -179,9 +192,7 @@ private static void runArray(String gender, List states) .addNamedParameter("gender", QueryParameterValue.string(gender)) .addNamedParameter( "states", - QueryParameterValue.array( - states.toArray(new String[]{}), - String.class)) + QueryParameterValue.array(states, String.class)) // Standard SQL syntax is required for parameterized queries. // See: https://cloud.google.com/bigquery/sql-reference/ .setUseLegacySql(false)