Merge pull request #1074 from finos/datetime-fixes

Use local time for column/row headers and computed functions

Author: texodus

cpp/perspective/CMakeLists.txt

@@ -45,21 +45,26 @@ function (psp_build_dep name cmake_file)
 		message(WARNING "${Cyan}Dependency found - not rebuilding - ${CMAKE_BINARY_DIR}/${name}-build${ColorReset}")
 	else()
 		configure_file(${cmake_file} ${name}-download/CMakeLists.txt)
+
 		execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
 			RESULT_VARIABLE result
 			WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${name}-download )
+
 		if(result)
 			message(FATAL_ERROR "CMake step for ${name} failed: ${result}")
 		endif()
+
 		execute_process(COMMAND ${CMAKE_COMMAND} --build .
 			RESULT_VARIABLE result
 			WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${name}-download )
+
 		if(result)
 			message(FATAL_ERROR "Build step for ${name} failed: ${result}")
 		endif()
 	endif()
 
 	if(${name} STREQUAL arrow)
+		# Overwrite arrow's CMakeLists with our custom, minimal one
 		configure_file(${PSP_CMAKE_MODULE_PATH}/arrow/CMakeLists.txt ${CMAKE_BINARY_DIR}/arrow-src/cpp/ COPYONLY)
 		configure_file(${PSP_CMAKE_MODULE_PATH}/arrow/config.h ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/ COPYONLY)
 		add_subdirectory(${CMAKE_BINARY_DIR}/arrow-src/cpp/
@@ -635,6 +640,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 		endif()
 
 		target_link_libraries(psp tbb)
+
 		target_link_libraries(binding tbb)
 
 		target_link_libraries(binding psp)

cpp/perspective/src/cpp/computed.cpp

@@ -1043,7 +1043,7 @@ t_computed_column::computed_functions = {
         {"category", "FunctionTokenType"},
         {"num_params", "1"},
         {"format_function", "x => `hour_of_day(${x})`"},
-        {"help", "Returns the hour of day (0-23) in UTC for the datetime column."},
+        {"help", "Returns the hour of day (0-23) for the datetime column."},
         {"signature", "hour_of_day(x: Datetime): Number"}
     }},
     {"day_of_week", {
@@ -1056,7 +1056,7 @@ t_computed_column::computed_functions = {
         {"category", "FunctionTokenType"},
         {"num_params", "1"},
         {"format_function", "x => `day_of_week(${x})`"},
-        {"help", "Returns the day of week in UTC for the datetime column."},
+        {"help", "Returns the day of week for the datetime column."},
         {"signature", "day_of_week(x: Datetime): String"}
     }},
     {"month_of_year", {
@@ -1069,7 +1069,7 @@ t_computed_column::computed_functions = {
         {"category", "FunctionTokenType"},
         {"num_params", "1"},
         {"format_function", "x => `month_of_year(${x})`"},
-        {"help", "Returns the month of year in UTC for the datetime column."},
+        {"help", "Returns the month of year for the datetime column."},
         {"signature", "month_of_year(x: Datetime): String"}
     }},
     {"second_bucket", {

cpp/perspective/src/cpp/computed_function.cpp

@@ -527,14 +527,13 @@ t_tscalar hour_of_day<DTYPE_TIME>(t_tscalar x) {
     // Convert the timestamp to a `sys_time` (alias for `time_point`)
     date::sys_time<std::chrono::milliseconds> ts(timestamp);
 
-    // Create a copy of the timestamp with day precision
-    date::sys_days days = date::floor<date::days>(ts);
-
-    // Subtract the day-precision `time_point` from the datetime-precision one
-    auto time_of_day = date::make_time(ts - days);
+    // Use localtime so that the hour of day is consistent with all output
+    // datetimes, which are in local time
+    std::time_t temp = std::chrono::system_clock::to_time_t(ts);
+    std::tm* t = std::localtime(&temp);
 
-    // Get the hour from the resulting `time_point`
-    rval.set(static_cast<std::int64_t>(time_of_day.hours().count()));
+    // Get the hour from the resulting `std::tm`
+    rval.set(static_cast<std::int64_t>(t->tm_hour));
     return rval;
 }
 
@@ -831,14 +830,14 @@ void day_of_week<DTYPE_TIME>(
     // Convert the timestamp to a `sys_time` (alias for `time_point`)
     date::sys_time<std::chrono::milliseconds> ts(timestamp);
 
-    // Create a copy of the timestamp with day precision
-    auto days = date::floor<date::days>(ts);
-
-    // Find the weekday and write it to the output column
-    auto weekday = date::year_month_weekday(days).weekday_indexed().weekday();
+    // Use localtime so that the hour of day is consistent with all output
+    // datetimes, which are in local time
+    std::time_t temp = std::chrono::system_clock::to_time_t(ts);
+    std::tm* t = std::localtime(&temp);
 
+    // Get the weekday from the resulting `std::tm`
     output_column->set_nth(
-        idx, days_of_week[(weekday - date::Sunday).count()]);
+        idx, days_of_week[t->tm_wday]);
 }
 
 template <>
@@ -871,18 +870,16 @@ void month_of_year<DTYPE_TIME>(
     // Convert the timestamp to a `sys_time` (alias for `time_point`)
     date::sys_time<std::chrono::milliseconds> ts(timestamp);
 
-    // Create a copy of the timestamp with day precision
-    auto days = date::floor<date::days>(ts);
-
-    // Cast the `time_point` to contain year/month/day
-    auto ymd = date::year_month_day(days);
+    // Use localtime so that the hour of day is consistent with all output
+    // datetimes, which are in local time
+    std::time_t temp = std::chrono::system_clock::to_time_t(ts);
+    std::tm* t = std::localtime(&temp);
 
-    // Get the month as an integer from 0 to 11
-    auto month = (ymd.month() -  date::January).count();
+    // Get the month from the resulting `std::tm`
+    auto month = t->tm_mon;
 
     // Get the month string and write into the output column
-    std::string month_of_year = months_of_year[month];
-    output_column->set_nth(idx, month_of_year);
+    output_column->set_nth(idx, months_of_year[month]);
 }
 
 } // end namespace computed_function

cpp/perspective/src/cpp/scalar.cpp

@@ -679,9 +679,23 @@ t_tscalar::to_string(bool for_expr) const {
             return ss.str();
         } break;
         case DTYPE_TIME: {
+            // Convert a millisecond UTC timestamp to a formatted datestring in
+            // local time, as all datetimes exported to the user happens in
+            // local time and not UTC.
             std::chrono::milliseconds timestamp(to_int64());
             date::sys_time<std::chrono::milliseconds> ts(timestamp);
-            ss << date::format("%Y-%m-%d %H:%M:%S", ts);
+            std::time_t temp = std::chrono::system_clock::to_time_t(ts);
+            std::tm* t = std::localtime(&temp);
+
+            // use a mix of std::put_time and date::format to properly
+            // represent datetimes to millisecond precision
+            ss << std::put_time(t, "%Y-%m-%d %H:%M:"); // ymd h:m
+
+            // TODO: we currently can't print out millisecond precision, but
+            // we need to.
+            ss << date::format("%S", ts); // represent second and millisecond
+            ss << std::put_time(t, " %Z"); // timezone
+
             return ss.str();
         } break;
         case DTYPE_STR: {

cpp/perspective/src/include/perspective/base.h

@@ -30,6 +30,7 @@
 #include <fstream>
 #include <boost/unordered_map.hpp>
 #include <perspective/portable.h>
+#include <stdlib.h>
 
 namespace perspective {
 

docs/i18n/en.json

@@ -35,15 +35,6 @@
       "node_modules/react/README": {
         "title": "node_modules/react/README"
       },
-      "obj/perspective-python": {
-        "title": "perspective-python API"
-      },
-      "obj/perspective-viewer": {
-        "title": "perspective-viewer API"
-      },
-      "obj/perspective": {
-        "title": "perspective API"
-      },
       "README": {
         "title": "README"
       }

docs/md/python.md

@@ -159,18 +159,18 @@ two methods into your object:
 
 #### Time Zone Handling
 
-When passing in `datetime` objects, Perspective checks the `tzinfo` attribute
-to see if a time zone is set. For more details, see this in-depth [explanation](https://github.com/finos/perspective/pull/867)
-of `perspective-python` semantics around time zone handling.
+Columns with the `datetime` type are stored internally as UTC timestamps in milliseconds since epoch (Unix Time),
+and are serialized to the user as `datetime.datetime` objects in _local time_ according to the Python runtime.
 
-##### Naive Datetimes
+Both ["naive" and "aware" datetimes](https://docs.python.org/3/library/datetime.html#aware-and-naive-objects) will be
+serialized to local time by Perspective, with the conversion determined by the `tzinfo` attribute:
 
-Objects with an unset `tzinfo` attribute (naive datetimes) are treated as _local time_, and do not undergo any time zone conversion.
+- "Naive" datetimes are assumed to be already in local time and are serialized as-is.
+- "Aware" datetimes will be converted to UTC from their original timezone, and then converted to local time
+from UTC.
 
-##### Aware Datetimes
-
-Objects with the `tzinfo` attribute set (aware datetimes) will be _converted into UTC_ before being stored in
-Perspective, and they will be _serialized as local time_.
+This behavior is consistent with Perspective's behavior in Javascript. For more details, see this
+in-depth [explanation](https://github.com/finos/perspective/pull/867) of `perspective-python` semantics around time zone handling.
 
 ##### Pandas Timestamps
 

packages/perspective-viewer/README.md

@@ -166,19 +166,19 @@ Sets new computed columns for the viewer.
 **Params**
 
 - computed-columns <code>[ &#x27;Array&#x27; ].&lt;Object&gt;</code> - An Array of computed column objects,
-which have three properties: `name`, a column name for the new column,
+which have three properties: `column`, a column name for the new column,
 `computed_function_name`, a String representing the computed function to
 apply, and `inputs`, an Array of String column names to be used as
 inputs to the computation.
 
 **Example** *(via Javascript DOM)*  
 ```js
 let elem = document.getElementById('my_viewer');
-elem.setAttribute('computed-columns', JSON.stringify([{name: "x+y", computed_function_name: "+", inputs: ["x", "y"]}]));
+elem.setAttribute('computed-columns', JSON.stringify([{column: "x+y", computed_function_name: "+", inputs: ["x", "y"]}]));
 ```
 **Example** *(via HTML)*  
 ```js
-<perspective-viewer computed-columns="[{name:'x+y',computed_function_name:'+',inputs:['x','y']}]""></perspective-viewer>
+<perspective-viewer computed-columns="[{column:'x+y',computed_function_name:'+',inputs:['x','y']}]""></perspective-viewer>
 ```
 
 * * *

packages/perspective-viewer/src/js/autocomplete_widget.js

@@ -218,8 +218,7 @@ class PerspectiveAutocompleteWidget extends HTMLElement {
 
         if (idx > -1 && children.length > 0) {
             children[idx].setAttribute("aria-selected", "true");
-            children[idx].scrollIntoView(true, {
-                behavior: "smooth",
+            children[idx].scrollIntoView({
                 block: "nearest"
             });
 

packages/perspective-viewer/src/js/viewer.js

@@ -182,16 +182,16 @@ class PerspectiveViewer extends ActionElement {
      * @kind member
      * @type {Array<Object>}
      * @param {Array<Object>} computed-columns An Array of computed column objects,
-     * which have three properties: `name`, a column name for the new column,
+     * which have three properties: `column`, a column name for the new column,
      * `computed_function_name`, a String representing the computed function to
      * apply, and `inputs`, an Array of String column names to be used as
      * inputs to the computation.
      * @fires PerspectiveViewer#perspective-config-update
      * @example <caption>via Javascript DOM</caption>
      * let elem = document.getElementById('my_viewer');
-     * elem.setAttribute('computed-columns', JSON.stringify([{name: "x+y", computed_function_name: "+", inputs: ["x", "y"]}]));
+     * elem.setAttribute('computed-columns', JSON.stringify([{column: "x+y", computed_function_name: "+", inputs: ["x", "y"]}]));
      * @example <caption>via HTML</caption>
-     * <perspective-viewer computed-columns="[{name:'x+y',computed_function_name:'+',inputs:['x','y']}]""></perspective-viewer>
+     * <perspective-viewer computed-columns="[{column:'x+y',computed_function_name:'+',inputs:['x','y']}]""></perspective-viewer>
      */
     @array_attribute
     "computed-columns"(computed_columns) {