feat(resampling): support setting custom dates for the start and end of extrapolation

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Version 2.1.0 - New feature release - 2024-12
+### Resampling recipe
+- :date: Support selecting custom dates for the start and end of extrapolation
+
 ## Version 2.0.5 - Bugfix release - 2025-01
 - :bug: Windowing would fail on quarterly and yearly frequencies
 

custom-recipes/timeseries-preparation-resampling/recipe.json

@@ -268,6 +268,58 @@
       ],
       "defaultValue": "clip"
     },
+    {
+      "name": "start_date_mode",
+      "label": "Extrapolation start date",
+      "type": "SELECT",
+      "selectChoices": [
+        {
+          "value": "AUTO",
+          "label": "First available"
+        },
+        {
+          "value": "CUSTOM",
+          "label": "Custom"
+        }
+      ],
+      "mandatory": false,
+      "defaultValue": "AUTO",
+      "visibilityCondition": "model.extrapolation_method == 'clip' || (model.extrapolation_method == 'interpolation' && model.interpolation_method != 'none')"
+    },
+    {
+      "name": "custom_start_date",
+      "label": "Custom start date",
+      "type": "DATE",
+      "mandatory": false,
+      "visibilityCondition": "(model.extrapolation_method == 'clip' || (model.extrapolation_method == 'interpolation' && model.interpolation_method != 'none')) && model.start_date_mode == 'CUSTOM'",
+      "description": "Set a custom date to start extrapolation from."
+    },
+    {
+      "name": "end_date_mode",
+      "label": "Extrapolation end date",
+      "type": "SELECT",
+      "selectChoices": [
+        {
+          "value": "AUTO",
+          "label": "Last available"
+        },
+        {
+          "value": "CUSTOM",
+          "label": "Custom"
+        }
+      ],
+      "mandatory": false,
+      "defaultValue": "AUTO",
+      "visibilityCondition": "model.extrapolation_method == 'clip' || (model.extrapolation_method == 'interpolation' && model.interpolation_method != 'none')"
+    },
+    {
+      "name": "custom_end_date",
+      "label": "Custom end date",
+      "type": "DATE",
+      "mandatory": false,
+      "visibilityCondition": "(model.extrapolation_method == 'clip' || (model.extrapolation_method == 'interpolation' && model.interpolation_method != 'none')) && model.end_date_mode == 'CUSTOM'",
+      "description": "Set a custom date to end extrapolation on."
+    },
     {
       "name": "category_imputation_method",
       "label": "Impute categorical data",

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "timeseries-preparation",
-    "version": "2.0.5",
+    "version": "2.1.0",
     "meta": {
         "supportLevel": "SUPPORTED",
         "label": "Time Series Preparation",

python-lib/dku_timeseries/resampling.py

@@ -28,7 +28,9 @@ def __init__(self,
                  time_unit_end_of_week="SUN",
                  clip_start=0,
                  clip_end=0,
-                 shift=0):
+                 shift=0,
+                 custom_start_date=None,
+                 custom_end_date=None):
 
         self.interpolation_method = interpolation_method
         self.extrapolation_method = extrapolation_method
@@ -42,6 +44,8 @@ def __init__(self,
         self.clip_start = reformat_time_value(float(clip_start), time_unit)
         self.clip_end = reformat_time_value(float(clip_end), time_unit)
         self.shift = reformat_time_value(float(shift), time_unit)
+        self.custom_start_date = custom_start_date
+        self.custom_end_date = custom_end_date
 
     def check(self):
 
@@ -113,6 +117,10 @@ def transform(self, df, datetime_column, groupby_columns=None):
 
         return df_resampled
 
+    def _can_customize_resampling_dates(self):
+        return self.params.extrapolation_method == 'clip' or (self.params.extrapolation_method == 'interpolation' and self.params.interpolation_method != 'none')
+
+
     def _compute_full_time_index(self, df, datetime_column):
         """
         From the resampling config, create the full index of the output dataframe.
@@ -125,6 +133,18 @@ def _compute_full_time_index(self, df, datetime_column):
         frequency = self.params.resampling_step
         time_step = self.params.time_step
         time_unit = self.params.time_unit
+
+        if self._can_customize_resampling_dates():
+            custom_start_date = self.params.custom_start_date
+            if custom_start_date:
+                if custom_start_date < start_time:
+                    start_time = custom_start_date
+
+            custom_end_date = self.params.custom_end_date
+            if custom_end_date:
+                if custom_end_date > end_time:
+                    end_time = custom_end_date
+
         return generate_date_range(start_time, end_time, clip_start, clip_end, shift, frequency, time_step, time_unit)
 
     def _resample(self, df, datetime_column, columns_to_resample, category_columns, reference_time_index, df_id=''):

python-lib/recipe_config_loading.py

@@ -1,5 +1,7 @@
 import sys
 
+import pandas as pd
+
 from dku_config.stl_config import STLConfig
 from dku_input_validator.decomposition_input_validator import DecompositionInputValidator
 from dku_timeseries import ExtremaExtractorParams
@@ -17,6 +19,24 @@ def get_resampling_params(recipe_config):
     def _p(param_name, default=None):
         return recipe_config.get(param_name, default)
 
+    def date_from_naive_datetime(datetime_str):
+        if not datetime_str:
+            return None
+
+        # the frontend sends a zulu datetime after offsetting it from midnight on the selected day
+        # eg. selecting 2025-01-31 in a browser set to UTC+4 sends "2025-01-30T20:00:00Z"
+        # while a browser set to UTC-4 sends "2025-01-31T04:00:00Z"
+        date = pd.Timestamp(datetime_str)
+
+        # round UTC+12 dates up (includes NZST), UTC-12 is not used anywhere
+        if date.hour == 12:
+            return date.ceil("D")
+        # round UTC+13 dates up (includes NZDT and the state of Samoa), without timezone info UTC-11 looks the same as UTC+13 but the latter only has very few inhabitants (mainly American Samoa)
+        if date.hour == 11:
+            logger.warning("The input date is either UTC-11 or UTC+13, it will be processed as UTC+13")
+            return date.ceil("D")
+        return date.round("D")
+
     interpolation_method = _p('interpolation_method')
     extrapolation_method = _p('extrapolation_method')
     constant_value = _p('constant_value')
@@ -28,6 +48,10 @@ def _p(param_name, default=None):
     clip_start = _p('clip_start')
     clip_end = _p('clip_end')
     shift = _p('shift')
+    start_date_mode = _p('start_date_mode', 'AUTO')
+    custom_start_date = date_from_naive_datetime(_p('custom_start_date')) if start_date_mode == 'CUSTOM' else None
+    end_date_mode = _p('end_date_mode', 'AUTO')
+    custom_end_date = date_from_naive_datetime(_p('custom_end_date')) if end_date_mode == 'CUSTOM' else None
 
     params = ResamplerParams(interpolation_method=interpolation_method,
                              extrapolation_method=extrapolation_method,
@@ -39,7 +63,9 @@ def _p(param_name, default=None):
                              time_unit_end_of_week=time_unit_end_of_week,
                              clip_start=clip_start,
                              clip_end=clip_end,
-                             shift=shift)
+                             shift=shift,
+                             custom_start_date=custom_start_date,
+                             custom_end_date=custom_end_date)
     params.check()
     return params
 

tests/python/unit/dku_timeseries/resampling/test_resampling.py

@@ -378,6 +378,20 @@ def test_no_extrapolation_week(self):
                                                                                     '2021-04-17T22:00:00.000000000', '2021-04-24T22:00:00.000000000',
                                                                                     '2021-05-01T22:00:00.000000000']))
 
+    def test_extrapolation_custom_dates(self):
+        data = [random.random() for _ in range(3)]
+        start_time = pd.Timestamp('2024-12-01T00:00:00Z')
+        df = _make_df_with_one_col(data, period=pd.DateOffset(days=1), start_time=start_time)
+        params = ResamplerParams(time_unit="days",
+                                 extrapolation_method='clip',
+                                 custom_start_date=pd.Timestamp("2024-11-30T00:00:00Z"),
+                                 custom_end_date=pd.Timestamp("2024-12-05T00:00:00Z"))
+        resampler = Resampler(params)
+        output_df = resampler.transform(df, TIME_COL)
+        np.testing.assert_array_equal(output_df[TIME_COL].values, pd.DatetimeIndex(['2024-11-30T00:00:00.000000000', '2024-12-01T00:00:00.000000000',
+                                                                                    '2024-12-02T00:00:00.000000000', '2024-12-03T00:00:00.000000000',
+                                                                                    '2024-12-04T00:00:00.000000000', '2024-12-05T00:00:00.000000000']))
+
     def test_no_extrapolation_long_format(self, long_format_df):
         params = ResamplerParams(time_unit="days", extrapolation_method='no_extrapolation')
         resampler = Resampler(params)

tests/python/unit/dku_timeseries/resampling/test_resampling_params.py

@@ -1,3 +1,4 @@
+import pandas as pd
 import pytest
 
 from recipe_config_loading import get_resampling_params
@@ -7,7 +8,8 @@
 def config():
     config = {u'clip_end': 0, u'extrapolation_method': u'none', u'shift': 0,
               u'datetime_column': u'Date', u'advanced_activated': False, u'time_unit': u'quarters', u'clip_start': 0, u'time_step': 2,
-              u'interpolation_method': u'linear'}
+              u'interpolation_method': u'linear',
+              'start_date_mode': 'AUTO', 'end_date_mode': 'AUTO'}
     return config
 
 
@@ -60,3 +62,31 @@ def test_no_category_imputation(self, config):
     def test_no_category_constant_value(self, config):
         params = get_resampling_params(config)
         assert params.category_constant_value == ""
+
+    def test_extrapolation_dates_params(self, config):
+        config["start_date_mode"] = "AUTO"
+        config["custom_start_date"] = "not read"
+        params = get_resampling_params(config)
+        assert params.custom_start_date is None
+
+        config["start_date_mode"] = "CUSTOM"
+
+        # UTC-11, rounds one day later than selected but that's required to support UTC+13
+        config["custom_start_date"] = "2025-01-31T11:00:00Z"
+        params = get_resampling_params(config)
+        assert params.custom_start_date == pd.Timestamp("2025-02-01T00:00:00Z")
+
+        # UTC-10
+        config["custom_start_date"] = "2025-01-31T10:00:00Z"
+        params = get_resampling_params(config)
+        assert params.custom_start_date == pd.Timestamp("2025-01-31T00:00:00Z")
+
+        # UTC+12
+        config["custom_start_date"] = "2025-01-30T12:00:00Z"
+        params = get_resampling_params(config)
+        assert params.custom_start_date == pd.Timestamp("2025-01-31T00:00:00Z")
+
+        # UTC+13
+        config["custom_start_date"] = "2025-01-30T11:00:00Z"
+        params = get_resampling_params(config)
+        assert params.custom_start_date == pd.Timestamp("2025-01-31T00:00:00Z")

tests/python/unit/requirements.txt

@@ -1,7 +1,6 @@
-pytest
+pytest<8
 allure-pytest==2.8.29
 pandas==1.0.5; python_version < '3.9'
-pandas==1.1.5; python_version == '3.9'
-pandas==1.3.5; python_version > '3.9'
+pandas==1.3.5; python_version >= '3.9'
 numpy==1.21.6; python_version == '3.7'
 numpy==1.23.5; python_version > '3.7'