Merge branch 'master' into reinvent-wheel

Author: texodus

.eslintrc.json

@@ -33,7 +33,7 @@
             "camelcase": "off",
             "@typescript-eslint/camelcase": "off"
 
-        }    
+        }
     }, {
         "files": ["**/test/**/*.js"],
         "rules": {
@@ -42,7 +42,7 @@
                 "comments": 200,
                 "ignoreTrailingComments": true
             }]
-        }    
+        }
     }, {
         "files": ["**/*.md"],
         "rules": {
@@ -64,6 +64,7 @@
             "ignoreTrailingComments": true
         }],
         "@typescript-eslint/camelcase": "off",
+        "linebreak-style": ["error","unix"],
         "camelcase": "off",
         "no-const-assign": "error",
         "no-this-before-super": "error",

.gitattributes

@@ -1,4 +1,5 @@
 *.ipynb linguist-documentation
 docs/* linguist-documentation
 python/perspective/notebooks/* linguist-documentation
-python/perspective/docs/* linguist-documentation
+python/perspective/docs/* linguist-documentation
+*.js text eol=lf

azure-pipelines.yml

@@ -231,9 +231,11 @@ jobs:
       env:
         # Set `BOOST_ROOT` manually, as `BOOST_ROOT` is removed in the VM:
         # https://github.com/actions/virtual-environments/issues/687
-        BOOST_ROOT: "C:/hostedtoolcache/windows/Boost/1.69.0/"
-        BOOST_INCLUDEDIR: "C:/hostedtoolcache/windows/Boost/1.69.0/include"
-        BOOST_LIBRARYDIR: "C:/hostedtoolcache/windows/Boost/1.69.0/libs"
+        # 06/18/2020 - seems like boost got moved to `x86_64` inside
+        # the boost folder, which broke builds for a bit.
+        BOOST_ROOT: "C:/hostedtoolcache/windows/Boost/1.69.0/x86_64/"
+        BOOST_INCLUDEDIR: "C:/hostedtoolcache/windows/Boost/1.69.0/x86_64/include"
+        BOOST_LIBRARYDIR: "C:/hostedtoolcache/windows/Boost/1.69.0/x86_64/libs"
 
 
 - job: 'MacOS_Mojave'

cpp/perspective/CMakeLists.txt

@@ -45,21 +45,26 @@ function (psp_build_dep name cmake_file)
 		message(WARNING "${Cyan}Dependency found - not rebuilding - ${CMAKE_BINARY_DIR}/${name}-build${ColorReset}")
 	else()
 		configure_file(${cmake_file} ${name}-download/CMakeLists.txt)
+
 		execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
 			RESULT_VARIABLE result
 			WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${name}-download )
+
 		if(result)
 			message(FATAL_ERROR "CMake step for ${name} failed: ${result}")
 		endif()
+
 		execute_process(COMMAND ${CMAKE_COMMAND} --build .
 			RESULT_VARIABLE result
 			WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${name}-download )
+
 		if(result)
 			message(FATAL_ERROR "Build step for ${name} failed: ${result}")
 		endif()
 	endif()
 
 	if(${name} STREQUAL arrow)
+		# Overwrite arrow's CMakeLists with our custom, minimal one
 		configure_file(${PSP_CMAKE_MODULE_PATH}/arrow/CMakeLists.txt ${CMAKE_BINARY_DIR}/arrow-src/cpp/ COPYONLY)
 		configure_file(${PSP_CMAKE_MODULE_PATH}/arrow/config.h ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/ COPYONLY)
 		add_subdirectory(${CMAKE_BINARY_DIR}/arrow-src/cpp/
@@ -297,7 +302,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 			# must be set to `NEW` to allow BOOST_ROOT to be defined by env var
 			cmake_policy(SET CMP0074 NEW)
 
-			if(DEFINED(ENV{BOOST_ROOT}))
+			if(DEFINED (ENV{BOOST_ROOT}))
 				set(Boost_NO_BOOST_CMAKE TRUE)
 
 				message(WARNING "${Cyan}BOOST_ROOT: $ENV{BOOST_ROOT} ${ColorReset}")
@@ -635,6 +640,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 		endif()
 
 		target_link_libraries(psp tbb)
+
 		target_link_libraries(binding tbb)
 
 		target_link_libraries(binding psp)

cpp/perspective/src/cpp/computed.cpp

@@ -1043,7 +1043,7 @@ t_computed_column::computed_functions = {
         {"category", "FunctionTokenType"},
         {"num_params", "1"},
         {"format_function", "x => `hour_of_day(${x})`"},
-        {"help", "Returns the hour of day (0-23) in UTC for the datetime column."},
+        {"help", "Returns the hour of day (0-23) for the datetime column."},
         {"signature", "hour_of_day(x: Datetime): Number"}
     }},
     {"day_of_week", {
@@ -1056,7 +1056,7 @@ t_computed_column::computed_functions = {
         {"category", "FunctionTokenType"},
         {"num_params", "1"},
         {"format_function", "x => `day_of_week(${x})`"},
-        {"help", "Returns the day of week in UTC for the datetime column."},
+        {"help", "Returns the day of week for the datetime column."},
         {"signature", "day_of_week(x: Datetime): String"}
     }},
     {"month_of_year", {
@@ -1069,7 +1069,7 @@ t_computed_column::computed_functions = {
         {"category", "FunctionTokenType"},
         {"num_params", "1"},
         {"format_function", "x => `month_of_year(${x})`"},
-        {"help", "Returns the month of year in UTC for the datetime column."},
+        {"help", "Returns the month of year for the datetime column."},
         {"signature", "month_of_year(x: Datetime): String"}
     }},
     {"second_bucket", {

cpp/perspective/src/cpp/computed_function.cpp

@@ -527,14 +527,13 @@ t_tscalar hour_of_day<DTYPE_TIME>(t_tscalar x) {
     // Convert the timestamp to a `sys_time` (alias for `time_point`)
     date::sys_time<std::chrono::milliseconds> ts(timestamp);
 
-    // Create a copy of the timestamp with day precision
-    date::sys_days days = date::floor<date::days>(ts);
-
-    // Subtract the day-precision `time_point` from the datetime-precision one
-    auto time_of_day = date::make_time(ts - days);
+    // Use localtime so that the hour of day is consistent with all output
+    // datetimes, which are in local time
+    std::time_t temp = std::chrono::system_clock::to_time_t(ts);
+    std::tm* t = std::localtime(&temp);
 
-    // Get the hour from the resulting `time_point`
-    rval.set(static_cast<std::int64_t>(time_of_day.hours().count()));
+    // Get the hour from the resulting `std::tm`
+    rval.set(static_cast<std::int64_t>(t->tm_hour));
     return rval;
 }
 
@@ -831,14 +830,14 @@ void day_of_week<DTYPE_TIME>(
     // Convert the timestamp to a `sys_time` (alias for `time_point`)
     date::sys_time<std::chrono::milliseconds> ts(timestamp);
 
-    // Create a copy of the timestamp with day precision
-    auto days = date::floor<date::days>(ts);
-
-    // Find the weekday and write it to the output column
-    auto weekday = date::year_month_weekday(days).weekday_indexed().weekday();
+    // Use localtime so that the hour of day is consistent with all output
+    // datetimes, which are in local time
+    std::time_t temp = std::chrono::system_clock::to_time_t(ts);
+    std::tm* t = std::localtime(&temp);
 
+    // Get the weekday from the resulting `std::tm`
     output_column->set_nth(
-        idx, days_of_week[(weekday - date::Sunday).count()]);
+        idx, days_of_week[t->tm_wday]);
 }
 
 template <>
@@ -871,18 +870,16 @@ void month_of_year<DTYPE_TIME>(
     // Convert the timestamp to a `sys_time` (alias for `time_point`)
     date::sys_time<std::chrono::milliseconds> ts(timestamp);
 
-    // Create a copy of the timestamp with day precision
-    auto days = date::floor<date::days>(ts);
-
-    // Cast the `time_point` to contain year/month/day
-    auto ymd = date::year_month_day(days);
+    // Use localtime so that the hour of day is consistent with all output
+    // datetimes, which are in local time
+    std::time_t temp = std::chrono::system_clock::to_time_t(ts);
+    std::tm* t = std::localtime(&temp);
 
-    // Get the month as an integer from 0 to 11
-    auto month = (ymd.month() -  date::January).count();
+    // Get the month from the resulting `std::tm`
+    auto month = t->tm_mon;
 
     // Get the month string and write into the output column
-    std::string month_of_year = months_of_year[month];
-    output_column->set_nth(idx, month_of_year);
+    output_column->set_nth(idx, months_of_year[month]);
 }
 
 } // end namespace computed_function

cpp/perspective/src/cpp/scalar.cpp

@@ -679,9 +679,23 @@ t_tscalar::to_string(bool for_expr) const {
             return ss.str();
         } break;
         case DTYPE_TIME: {
+            // Convert a millisecond UTC timestamp to a formatted datestring in
+            // local time, as all datetimes exported to the user happens in
+            // local time and not UTC.
             std::chrono::milliseconds timestamp(to_int64());
             date::sys_time<std::chrono::milliseconds> ts(timestamp);
-            ss << date::format("%Y-%m-%d %H:%M:%S", ts);
+            std::time_t temp = std::chrono::system_clock::to_time_t(ts);
+            std::tm* t = std::localtime(&temp);
+
+            // use a mix of std::put_time and date::format to properly
+            // represent datetimes to millisecond precision
+            ss << std::put_time(t, "%Y-%m-%d %H:%M:"); // ymd h:m
+
+            // TODO: we currently can't print out millisecond precision, but
+            // we need to.
+            ss << date::format("%S", ts); // represent second and millisecond
+            ss << std::put_time(t, " %Z"); // timezone
+
             return ss.str();
         } break;
         case DTYPE_STR: {

cpp/perspective/src/include/perspective/base.h

@@ -30,6 +30,7 @@
 #include <fstream>
 #include <boost/unordered_map.hpp>
 #include <perspective/portable.h>
+#include <stdlib.h>
 
 namespace perspective {
 

docs/i18n/en.json

@@ -35,15 +35,6 @@
       "node_modules/react/README": {
         "title": "node_modules/react/README"
       },
-      "obj/perspective-python": {
-        "title": "perspective-python API"
-      },
-      "obj/perspective-viewer": {
-        "title": "perspective-viewer API"
-      },
-      "obj/perspective": {
-        "title": "perspective API"
-      },
       "README": {
         "title": "README"
       }

docs/md/python.md

@@ -159,18 +159,18 @@ two methods into your object:
 
 #### Time Zone Handling
 
-When passing in `datetime` objects, Perspective checks the `tzinfo` attribute
-to see if a time zone is set. For more details, see this in-depth [explanation](https://github.com/finos/perspective/pull/867)
-of `perspective-python` semantics around time zone handling.
+Columns with the `datetime` type are stored internally as UTC timestamps in milliseconds since epoch (Unix Time),
+and are serialized to the user as `datetime.datetime` objects in _local time_ according to the Python runtime.
 
-##### Naive Datetimes
+Both ["naive" and "aware" datetimes](https://docs.python.org/3/library/datetime.html#aware-and-naive-objects) will be
+serialized to local time by Perspective, with the conversion determined by the `tzinfo` attribute:
 
-Objects with an unset `tzinfo` attribute (naive datetimes) are treated as _local time_, and do not undergo any time zone conversion.
+- "Naive" datetimes are assumed to be already in local time and are serialized as-is.
+- "Aware" datetimes will be converted to UTC from their original timezone, and then converted to local time
+from UTC.
 
-##### Aware Datetimes
-
-Objects with the `tzinfo` attribute set (aware datetimes) will be _converted into UTC_ before being stored in
-Perspective, and they will be _serialized as local time_.
+This behavior is consistent with Perspective's behavior in Javascript. For more details, see this
+in-depth [explanation](https://github.com/finos/perspective/pull/867) of `perspective-python` semantics around time zone handling.
 
 ##### Pandas Timestamps