Skip to content
This repository was archived by the owner on Aug 2, 2022. It is now read-only.

Allow Timestamp/Datetime values to use up to 6 digits of microsecond precision #988

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,37 @@
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.time.temporal.ChronoUnit;
import lombok.RequiredArgsConstructor;

@RequiredArgsConstructor
public class ExprDatetimeValue extends AbstractExprValue {
private static final DateTimeFormatter formatter = DateTimeFormatter
.ofPattern("yyyy-MM-dd HH:mm:ss[.SSSSSS]");
private final LocalDateTime datetime;

private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS;
private static final int MIN_FRACTION_SECONDS = 0;
private static final int MAX_FRACTION_SECONDS = 6;

static {
FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd HH:mm:ss")
.appendFraction(
ChronoField.MICRO_OF_SECOND,
MIN_FRACTION_SECONDS,
MAX_FRACTION_SECONDS,
true)
.toFormatter();
}

/**
* Constructor with datetime string as input.
*/
public ExprDatetimeValue(String datetime) {
try {
this.datetime = LocalDateTime.parse(datetime, formatter);
this.datetime = LocalDateTime.parse(datetime, FORMATTER_VARIABLE_MICROS);
} catch (DateTimeParseException e) {
throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please "
+ "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", datetime));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@
import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.util.Objects;
import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;

/**
Expand All @@ -35,15 +35,30 @@
public class ExprTimeValue extends AbstractExprValue {
private final LocalTime time;

private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS;
private static final int MIN_FRACTION_SECONDS = 0;
private static final int MAX_FRACTION_SECONDS = 6;

static {
FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder()
.appendPattern("HH:mm:ss")
.appendFraction(
ChronoField.MICRO_OF_SECOND,
MIN_FRACTION_SECONDS,
MAX_FRACTION_SECONDS,
true)
.toFormatter();
}

/**
* Constructor.
*/
public ExprTimeValue(String time) {
try {
this.time = LocalTime.parse(time);
this.time = LocalTime.parse(time, FORMATTER_VARIABLE_MICROS);
} catch (DateTimeParseException e) {
throw new SemanticCheckException(String.format("time:%s in unsupported format, please use "
+ "HH:mm:ss", time));
+ "HH:mm:ss[.SSSSSS]", time));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.time.temporal.ChronoUnit;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
Expand All @@ -43,18 +45,33 @@ public class ExprTimestampValue extends AbstractExprValue {
/**
* todo. only support timestamp in format yyyy-MM-dd HH:mm:ss.
*/
private static final DateTimeFormatter FORMATTER = DateTimeFormatter
.ofPattern("yyyy-MM-dd HH:mm:ss[.SSSSSS]");
private static final DateTimeFormatter FORMATTER_WITNOUT_NANO = DateTimeFormatter
.ofPattern("yyyy-MM-dd HH:mm:ss");
private final Instant timestamp;

private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS;
private static final int MIN_FRACTION_SECONDS = 0;
private static final int MAX_FRACTION_SECONDS = 6;

static {
FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd HH:mm:ss")
.appendFraction(
ChronoField.MICRO_OF_SECOND,
MIN_FRACTION_SECONDS,
MAX_FRACTION_SECONDS,
true)
.toFormatter();
}

/**
* Constructor.
*/
public ExprTimestampValue(String timestamp) {
try {
this.timestamp = LocalDateTime.parse(timestamp, FORMATTER).atZone(ZONE).toInstant();
this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_MICROS)
.atZone(ZONE)
.toInstant();
} catch (DateTimeParseException e) {
throw new SemanticCheckException(String.format("timestamp:%s in unsupported format, please "
+ "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", timestamp));
Expand All @@ -66,7 +83,7 @@ public ExprTimestampValue(String timestamp) {
public String value() {
return timestamp.getNano() == 0 ? FORMATTER_WITNOUT_NANO.withZone(ZONE)
.format(timestamp.truncatedTo(ChronoUnit.SECONDS))
: FORMATTER.withZone(ZONE).format(timestamp);
: FORMATTER_VARIABLE_MICROS.withZone(ZONE).format(timestamp);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@

public class DateTimeValueTest {

private static final int MICROS_PRECISION_MAX = 6;

@Test
public void timeValueInterfaceTest() {
ExprValue timeValue = new ExprTimeValue("01:01:01");
Expand Down Expand Up @@ -103,7 +105,7 @@ public void dateInUnsupportedFormat() {
public void timeInUnsupportedFormat() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class, () -> new ExprTimeValue("01:01:0"));
assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss",
assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSS]",
exception.getMessage());
}

Expand Down Expand Up @@ -172,7 +174,96 @@ public void stringTimeValue() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class,
() -> new ExprStringValue("01:01:0").timeValue());
assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss",
assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSS]",
exception.getMessage());
}

@Test
public void timeWithVariableMicroPrecision() {
String timeWithMicrosFormat = "10:11:12.%s";

// Check all lengths of microsecond precision, up to max precision accepted
StringBuilder micros = new StringBuilder();
for (int microPrecision = 1; microPrecision <= MICROS_PRECISION_MAX; microPrecision++) {
micros.append(microPrecision);
String timeWithMicros = String.format(timeWithMicrosFormat, micros);

ExprValue timeValue = new ExprTimeValue(timeWithMicros);
assertEquals(LocalTime.parse(timeWithMicros), timeValue.timeValue());
}
}

@Test
public void timestampWithVariableMicroPrecision() {
String dateValue = "2020-08-17";
String timeWithMicrosFormat = "10:11:12.%s";

// Check all lengths of microsecond precision, up to max precision accepted
StringBuilder micros = new StringBuilder();
for (int microPrecision = 1; microPrecision <= MICROS_PRECISION_MAX; microPrecision++) {
micros.append(microPrecision);
String timeWithMicros = String.format(timeWithMicrosFormat, micros);

String timestampString = String.format("%s %s", dateValue, timeWithMicros);
ExprValue timestampValue = new ExprTimestampValue(timestampString);

assertEquals(LocalDate.parse(dateValue), timestampValue.dateValue());
assertEquals(LocalTime.parse(timeWithMicros), timestampValue.timeValue());
String localDateTime = String.format("%sT%s", dateValue, timeWithMicros);
assertEquals(LocalDateTime.parse(localDateTime), timestampValue.datetimeValue());
}
}

@Test
public void datetimeWithVariableMicroPrecision() {
String dateValue = "2020-08-17";
String timeWithMicrosFormat = "10:11:12.%s";

// Check all lengths of microsecond precision, up to max precision accepted
StringBuilder micros = new StringBuilder();
for (int microPrecision = 1; microPrecision <= MICROS_PRECISION_MAX; microPrecision++) {
micros.append(microPrecision);
String timeWithMicros = String.format(timeWithMicrosFormat, micros);

String datetimeString = String.format("%s %s", dateValue, timeWithMicros);
ExprValue datetimeValue = new ExprDatetimeValue(datetimeString);

assertEquals(LocalDate.parse(dateValue), datetimeValue.dateValue());
assertEquals(LocalTime.parse(timeWithMicros), datetimeValue.timeValue());
String localDateTime = String.format("%sT%s", dateValue, timeWithMicros);
assertEquals(LocalDateTime.parse(localDateTime), datetimeValue.datetimeValue());
}
}

@Test
public void timestampOverMaxMicroPrecision() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class,
() -> new ExprTimestampValue("2020-07-07 01:01:01.1234567"));
assertEquals(
"timestamp:2020-07-07 01:01:01.1234567 in unsupported format, "
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
exception.getMessage());
}

@Test
public void datetimeOverMaxMicroPrecision() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class,
() -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567"));
assertEquals(
"datetime:2020-07-07 01:01:01.1234567 in unsupported format, "
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
exception.getMessage());
}

@Test
public void timeOverMaxMicroPrecision() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class,
() -> new ExprTimeValue("01:01:01.1234567"));
assertEquals(
"time:01:01:01.1234567 in unsupported format, please use HH:mm:ss[.SSSSSS]",
exception.getMessage());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,7 @@ public void microsecond() {
expression = dsl.microsecond(DSL.literal(new ExprTimestampValue("2020-08-17 01:02:03.000010")));
assertEquals(INTEGER, expression.type());
assertEquals(integerValue(10), expression.valueOf(env));
assertEquals("microsecond(TIMESTAMP '2020-08-17 01:02:03.000010')", expression.toString());
assertEquals("microsecond(TIMESTAMP '2020-08-17 01:02:03.00001')", expression.toString());
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,20 +217,44 @@ public void testMicrosecond() throws IOException {
verifySchema(result, schema("f", null, "integer"));
verifySome(result.getJSONArray("datarows"), rows(123456));

// Explicit timestamp value with less than 6 microsecond digits
result = executeQuery(String.format(
"source=%s | eval f = microsecond(timestamp('2020-09-16 17:30:00.1234')) | fields f", TEST_INDEX_DATE));
verifySchema(result, schema("f", null, "integer"));
verifySome(result.getJSONArray("datarows"), rows(123400));

result = executeQuery(String.format(
"source=%s | eval f = microsecond(time('17:30:00.000010')) | fields f", TEST_INDEX_DATE));
verifySchema(result, schema("f", null, "integer"));
verifySome(result.getJSONArray("datarows"), rows(10));

// Explicit time value with less than 6 microsecond digits
result = executeQuery(String.format(
"source=%s | eval f = microsecond(time('17:30:00.1234')) | fields f", TEST_INDEX_DATE));
verifySchema(result, schema("f", null, "integer"));
verifySome(result.getJSONArray("datarows"), rows(123400));

result = executeQuery(String.format(
"source=%s | eval f = microsecond('2020-09-16 17:30:00.123456') | fields f", TEST_INDEX_DATE));
verifySchema(result, schema("f", null, "integer"));
verifySome(result.getJSONArray("datarows"), rows(123456));

// Implicit timestamp value with less than 6 microsecond digits
result = executeQuery(String.format(
"source=%s | eval f = microsecond('2020-09-16 17:30:00.1234') | fields f", TEST_INDEX_DATE));
verifySchema(result, schema("f", null, "integer"));
verifySome(result.getJSONArray("datarows"), rows(123400));

result = executeQuery(String.format(
"source=%s | eval f = microsecond('17:30:00.000010') | fields f", TEST_INDEX_DATE));
verifySchema(result, schema("f", null, "integer"));
verifySome(result.getJSONArray("datarows"), rows(10));

// Implicit time value with less than 6 microsecond digits
result = executeQuery(String.format(
"source=%s | eval f = microsecond('17:30:00.1234') | fields f", TEST_INDEX_DATE));
verifySchema(result, schema("f", null, "integer"));
verifySome(result.getJSONArray("datarows"), rows(123400));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,14 @@

package com.amazon.opendistroforelasticsearch.sql.sql;

import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT;
import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows;
import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema;
import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows;
import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema;
import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
import static com.amazon.opendistroforelasticsearch.sql.util.TestUtils.getResponseBody;

import com.amazon.opendistroforelasticsearch.sql.ast.expression.In;
import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils;
import com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase;
import com.amazon.opendistroforelasticsearch.sql.util.TestUtils;
Expand Down Expand Up @@ -251,17 +250,38 @@ public void testMicrosecond() throws IOException {
schema("microsecond(timestamp('2020-09-16 17:30:00.123456'))", null, "integer"));
verifyDataRows(result, rows(123456));

// Explicit timestamp value with less than 6 microsecond digits
result = executeQuery("select microsecond(timestamp('2020-09-16 17:30:00.1234'))");
verifySchema(result,
schema("microsecond(timestamp('2020-09-16 17:30:00.1234'))", null, "integer"));
verifyDataRows(result, rows(123400));

result = executeQuery("select microsecond(time('17:30:00.000010'))");
verifySchema(result, schema("microsecond(time('17:30:00.000010'))", null, "integer"));
verifyDataRows(result, rows(10));

// Explicit time value with less than 6 microsecond digits
result = executeQuery("select microsecond(time('17:30:00.1234'))");
verifySchema(result, schema("microsecond(time('17:30:00.1234'))", null, "integer"));
verifyDataRows(result, rows(123400));

result = executeQuery("select microsecond('2020-09-16 17:30:00.123456')");
verifySchema(result, schema("microsecond('2020-09-16 17:30:00.123456')", null, "integer"));
verifyDataRows(result, rows(123456));

// Implicit timestamp value with less than 6 microsecond digits
result = executeQuery("select microsecond('2020-09-16 17:30:00.1234')");
verifySchema(result, schema("microsecond('2020-09-16 17:30:00.1234')", null, "integer"));
verifyDataRows(result, rows(123400));

result = executeQuery("select microsecond('17:30:00.000010')");
verifySchema(result, schema("microsecond('17:30:00.000010')", null, "integer"));
verifyDataRows(result, rows(10));

// Implicit time value with less than 6 microsecond digits
result = executeQuery("select microsecond('17:30:00.1234')");
verifySchema(result, schema("microsecond('17:30:00.1234')", null, "integer"));
verifyDataRows(result, rows(123400));
}

@Test
Expand Down