Skip to content

Commit 37501ef

Browse files
dengdifandengdifan
authored
Tft forecating with features (automl#6)
* time feature transform * tft with time-variing features * transform features allowed for all architecture * repair mask for temporal fusion layer * maint
1 parent 34d556a commit 37501ef

File tree

15 files changed

+712
-320
lines changed

15 files changed

+712
-320
lines changed

‎autoPyTorch/api/time_series_forecasting.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ def search(
120120
target_variables: Optional[Union[Tuple[int], Tuple[str], np.ndarray]] = None,
121121
n_prediction_steps: int = 1,
122122
freq: Optional[Union[str, int, List[int]]] = None,
123+
start_times_train: List[pd.DatetimeIndex] = [],
124+
start_times_test: Optional[List[pd.DatetimeIndex]] = None,
123125
dataset_name: Optional[str] = None,
124126
budget_type: str = 'epochs',
125127
min_budget: Union[int, str] = 5,
@@ -266,12 +268,15 @@ def search(
266268
# Fit a input validator to check the provided data
267269
# Also, an encoder is fit to both train and test data,
268270
# to prevent unseen categories during inference
269-
self.InputValidator.fit(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)
271+
self.InputValidator.fit(X_train=X_train, y_train=y_train, start_times_train=start_times_train,
272+
X_test=X_test, y_test=y_test, start_times_test=start_times_test)
270273

271274
self.dataset = TimeSeriesForecastingDataset(
272275
X=X_train, Y=y_train,
273276
X_test=X_test, Y_test=y_test,
274277
freq=freq,
278+
start_times_train=start_times_train,
279+
start_times_test=start_times_test,
275280
validator=self.InputValidator,
276281
resampling_strategy=self.resampling_strategy,
277282
resampling_strategy_args=self.resampling_strategy_args,

‎autoPyTorch/configs/forecasting_init_cfgs.json

Lines changed: 64 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"data_loader:backcast": false,
55
"data_loader:sample_strategy": "SeqUniform",
66
"data_loader:num_batches_per_epoch": 50,
7+
"data_loader:transform_time_features": false,
78
"lr_scheduler:__choice__": "ReduceLROnPlateau",
89
"lr_scheduler:ReduceLROnPlateau:mode": "max",
910
"lr_scheduler:ReduceLROnPlateau:factor": 0.5,
@@ -130,35 +131,42 @@
130131
},
131132
"Seq2Seq-Transformer2Transformer": {
132133
"loss:__choice__": "DistributionLoss",
134+
"data_loader:transform_time_features": true,
133135
"loss:DistributionLoss:dist_cls": "studentT",
134136
"loss:DistributionLoss:forecast_strategy": "sample",
135137
"loss:DistributionLoss:aggregation": "median",
136138
"loss:DistributionLoss:num_samples": 100,
137-
"network_backbone:__choice__": "TransformerEncoder",
138-
"network_backbone:TransformerEncoder:d_model_log": 5,
139-
"network_backbone:TransformerEncoder:activation": "gelu",
140-
"network_backbone:TransformerEncoder:num_layers": 1,
141-
"network_backbone:TransformerEncoder:decoder_type": "TransformerDecoder",
142-
"network_backbone:TransformerEncoder:use_dropout": true,
143-
"network_backbone:TransformerEncoder:use_positional_encoder": true,
144-
"network_backbone:TransformerEncoder:dropout_positional_encoder": 0.1,
145-
"network_backbone:TransformerEncoder:d_feed_forward_log": 7,
146-
"network_backbone:TransformerEncoder:n_head_log": 3,
147-
"network_backbone:TransformerEncoder:layer_norm_eps": 1e-05,
148-
"network_backbone:TransformerEncoder:dropout": 0.1,
149-
"network_backbone:TransformerEncoder:use_layer_norm_output": true,
150-
"network_backbone:TransformerEncoder:layer_norm_eps_output": 1e-05,
151-
"network_backbone:TransformerDecoder:activation": "gelu",
152-
"network_backbone:TransformerDecoder:num_layers": 1,
153-
"network_backbone:TransformerDecoder:use_dropout": true,
154-
"network_backbone:TransformerDecoder:use_positional_decoder": true,
155-
"network_backbone:TransformerDecoder:dropout_positional_decoder": 0.1,
156-
"network_backbone:TransformerDecoder:d_feed_forward_log": 7,
157-
"network_backbone:TransformerDecoder:n_head_log": 3,
158-
"network_backbone:TransformerDecoder:layer_norm_eps": 1e-05,
159-
"network_backbone:TransformerDecoder:dropout": 0.1,
160-
"network_backbone:TransformerDecoder:use_layer_norm_output": true,
161-
"network_backbone:TransformerDecoder:layer_norm_eps_output": 1e-05
139+
"network_backbone:__choice__": "seq_encoder",
140+
"network_backbone:seq_encoder:skip_connection": false,
141+
"network_backbone:seq_encoder:num_blocks": 1,
142+
"network_backbone:seq_encoder:use_temporal_fusion": false,
143+
"network_backbone:seq_encoder:variable_selection": false,
144+
"network_backbone:seq_encoder:decoder_auto_regressive": true,
145+
"network_backbone:seq_encoder:block_1:__choice__": "TransformerEncoder",
146+
"network_backbone:seq_encoder:block_1:TransformerEncoder:d_model_log": 5,
147+
"network_backbone:seq_encoder:block_1:TransformerEncoder:activation": "gelu",
148+
"network_backbone:seq_encoder:block_1:TransformerEncoder:num_layers": 1,
149+
"network_backbone:seq_encoder:block_1:TransformerEncoder:decoder_type": "TransformerDecoder",
150+
"network_backbone:seq_encoder:block_1:TransformerEncoder:use_dropout": true,
151+
"network_backbone:seq_encoder:block_1:TransformerEncoder:use_positional_encoder": true,
152+
"network_backbone:seq_encoder:block_1:TransformerEncoder:dropout_positional_encoder": 0.1,
153+
"network_backbone:seq_encoder:block_1:TransformerEncoder:d_feed_forward_log": 7,
154+
"network_backbone:seq_encoder:block_1:TransformerEncoder:n_head_log": 3,
155+
"network_backbone:seq_encoder:block_1:TransformerEncoder:layer_norm_eps": 1e-05,
156+
"network_backbone:seq_encoder:block_1:TransformerEncoder:dropout": 0.1,
157+
"network_backbone:seq_encoder:block_1:TransformerEncoder:use_layer_norm_output": true,
158+
"network_backbone:seq_encoder:block_1:TransformerEncoder:layer_norm_eps_output": 1e-05,
159+
"network_backbone:seq_encoder:block_1:TransformerDecoder:activation": "gelu",
160+
"network_backbone:seq_encoder:block_1:TransformerDecoder:num_layers": 1,
161+
"network_backbone:seq_encoder:block_1:TransformerDecoder:use_dropout": true,
162+
"network_backbone:seq_encoder:block_1:TransformerDecoder:use_positional_decoder": true,
163+
"network_backbone:seq_encoder:block_1:TransformerDecoder:dropout_positional_decoder": 0.1,
164+
"network_backbone:seq_encoder:block_1:TransformerDecoder:d_feed_forward_log": 7,
165+
"network_backbone:seq_encoder:block_1:TransformerDecoder:n_head_log": 3,
166+
"network_backbone:seq_encoder:block_1:TransformerDecoder:layer_norm_eps": 1e-05,
167+
"network_backbone:seq_encoder:block_1:TransformerDecoder:dropout": 0.1,
168+
"network_backbone:seq_encoder:block_1:TransformerDecoder:use_layer_norm_output": true,
169+
"network_backbone:seq_encoder:block_1:TransformerDecoder:layer_norm_eps_output": 1e-05
162170
},
163171
"NBEATS-I": {
164172
"target_scaler:__choice__": "TargetNoScaler",
@@ -206,6 +214,37 @@
206214
"network_backbone:flat_encoder:NBEATSDecoder:weight_sharing_g": false,
207215
"network_backbone:flat_encoder:NBEATSDecoder:expansion_coefficient_length_g": 32,
208216
"network_backbone:flat_encoder:NBEATSDecoder:dropout_g": 0.1
217+
},
218+
"TemoporalFusionTransformer": {
219+
"loss:__choice__": "QuantileLoss",
220+
"target_scaler:__choice__": "TargetStandardScaler",
221+
"data_loader:transform_time_features": true,
222+
"loss:QuantileLoss:lower_quantile": 0.1,
223+
"loss:QuantileLoss:upper_quantile": 0.9,
224+
"network_backbone:__choice__": "seq_encoder",
225+
"network_backbone:seq_encoder:skip_connection": true,
226+
"network_backbone:seq_encoder:num_blocks": 1,
227+
"network_backbone:seq_encoder:variable_selection": true,
228+
"network_backbone:seq_encoder:share_single_variable_networks": false,
229+
"network_backbone:seq_encoder:skip_connection_type": "gate_add_norm",
230+
"network_backbone:seq_encoder:variable_selection_use_dropout": true,
231+
"network_backbone:seq_encoder:variable_selection_dropout_rate": 0.1,
232+
"network_backbone:seq_encoder:grn_use_dropout": true,
233+
"network_backbone:seq_encoder:grn_dropout_rate": 0.1,
234+
"network_backbone:seq_encoder:block_1:__choice__": "RNNEncoder",
235+
"network_backbone:seq_encoder:decoder_auto_regressive": false,
236+
"network_backbone:seq_encoder:block_1:RNNEncoder:cell_type": "lstm",
237+
"network_backbone:seq_encoder:block_1:RNNEncoder:num_layers": 1,
238+
"network_backbone:seq_encoder:block_1:RNNEncoder:hidden_size": 32,
239+
"network_backbone:seq_encoder:block_1:RNNEncoder:bidirectional": false,
240+
"network_backbone:seq_encoder:block_1:RNNEncoder:use_dropout": false,
241+
"network_backbone:seq_encoder:block_1:RNNEncoder:decoder_type": "RNNDecoder",
242+
"network_backbone:seq_encoder:block_1:RNNDecoder:decoder_type": "RNNDecoder",
243+
"network_backbone:seq_encoder:use_temporal_fusion": true,
244+
"network_backbone:seq_encoder:temporal_fusion:attention_d_model_log": 5,
245+
"network_backbone:seq_encoder:temporal_fusion:attention_n_head_log": 2,
246+
"network_backbone:seq_encoder:temporal_fusion:use_dropout": true,
247+
"network_backbone:seq_encoder:temporal_fusion:dropout_rate": 0.1
209248
}
210249
}
211250
}

‎autoPyTorch/constants_forecasting.py

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@
55
FORECASTING_BUDGET_TYPE = ['resolution', 'num_seq', 'num_sample_per_seq']
66

77
SEASONALITY_MAP = {
8-
"minutely": [1440, 10080, 525960],
9-
"10_minutes": [144, 1008, 52596],
10-
"half_hourly": [48, 336, 17532],
11-
"hourly": [24, 168, 8766],
12-
"daily": 7,
13-
"weekly": 365.25 / 7,
14-
"monthly": 12,
15-
"quarterly": 4,
16-
"yearly": 1
8+
"1min": [1440, 10080, 525960],
9+
"10min": [144, 1008, 52596],
10+
"30min": [48, 336, 17532],
11+
"1H": [24, 168, 8766],
12+
"1D": 7,
13+
"1W": 365.25 / 7,
14+
"1M": 12,
15+
"1Q": 4,
16+
"1Y": 1
1717
}
1818

1919
MAX_WINDOW_SIZE_BASE = 500

‎autoPyTorch/data/time_series_forecasting_validator.py

Lines changed: 48 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@
22

33
# -*- encoding: utf-8 -*-
44
import logging
5-
from typing import Optional, Tuple, List, Union
5+
import warnings
6+
from typing import Optional, Tuple, List, Union, Dict
67
import numpy as np
7-
8+
import pandas as pd
89
from sklearn.base import BaseEstimator
910
from sklearn.exceptions import NotFittedError
1011

@@ -23,6 +24,10 @@ def __init__(self,
2324
self._is_uni_variant = False
2425
self.known_future_features = None
2526
self.n_prediction_steps = 1
27+
self.start_times_train = None
28+
self.start_times_test = None
29+
self.feature_shapes = {}
30+
self.feature_names = []
2631

2732
"""
2833
A validator designed for a time series forecasting dataset.
@@ -35,10 +40,33 @@ def fit(
3540
y_train: SUPPORTED_TARGET_TYPES,
3641
X_test: Optional[SUPPORTED_FEAT_TYPES] = None,
3742
y_test: Optional[SUPPORTED_TARGET_TYPES] = None,
43+
start_times_train: Optional[List[pd.DatetimeIndex]] = None,
44+
start_times_test: Optional[List[pd.DatetimeIndex]] = None,
45+
freq: str = '1Y',
3846
n_prediction_steps: int = 1,
3947
known_future_features: Optional[List[Union[int, str]]] = None,
48+
use_time_features: bool = False
4049
) -> BaseEstimator:
4150
self.n_prediction_steps = n_prediction_steps
51+
52+
if y_test is not None and bool(start_times_test) != bool(start_times_train):
53+
warnings.warn('One of start_times_test or start_times_train is missing! This might result in the '
54+
'risk of not proper evaluated ')
55+
56+
if start_times_train is None:
57+
start_times_train = [pd.DatetimeIndex(pd.to_datetime(['1900-01-01']), freq=freq)] * len(y_train)
58+
else:
59+
assert len(start_times_train) == len(y_train), 'start_times_train must have the same length as y_train!'
60+
61+
if y_test is not None:
62+
if start_times_test is None:
63+
start_times_test = [pd.DatetimeIndex(pd.to_datetime(['1900-01-01']), freq=freq)] * len(y_test)
64+
else:
65+
assert len(start_times_train) == len(y_train), 'start_times_train must have the same length as y_train!'
66+
67+
self.start_times_train = start_times_train
68+
self.start_times_test = start_times_test
69+
4270
if X_train is None:
4371
self._is_uni_variant = True
4472
if self._is_uni_variant:
@@ -64,13 +92,22 @@ def fit(
6492
" {} for features and {} for targets".format(len(X_test), len(y_test), ))
6593
# TODO write a feature input validator to check X_test for known_future_features
6694
super().fit(X_train[0], y_train[0], X_test[0], y_test[0])
67-
else:
68-
super().fit(X_train[0], y_train[0])
95+
self.feature_validator.fit(X_train[0], None if X_test is None else X_test[0])
96+
self.target_validator.fit(y_train[0], None if y_test is None else y_test[0])
97+
self._is_fitted = True
6998

7099
self.check_input_shapes(X_train, y_train, is_training=True)
71100

72101
if X_test is not None:
73102
self.check_input_shapes(X_test, y_test, is_training=False)
103+
if hasattr(X_train[0], 'columns'):
104+
features = X_train[0].columns.values.tolist()
105+
else:
106+
features = list(map(str, range(len(X_train[0]))))
107+
for feature in features:
108+
self.feature_names.append(feature)
109+
self.feature_shapes[feature] = 1
110+
74111
return self
75112

76113
@staticmethod
@@ -125,20 +162,23 @@ def transform(
125162

126163
start_idx = 0
127164

165+
group_ids = np.arange(len(sequence_lengths)).repeat(sequence_lengths)
166+
128167
if self._is_uni_variant:
129-
y_flat = np.empty([num_data, num_targets])
168+
y_flat = pd.DataFrame(np.empty([num_data, num_targets]), index=group_ids)
130169
for seq_idx, seq_length in enumerate(sequence_lengths):
131170
end_idx = start_idx + seq_length
132171
y_flat[start_idx: end_idx] = np.array(y[seq_idx]).reshape([-1, num_targets])
133172
start_idx = end_idx
134-
y_transformed = self.target_validator.transform(y_flat) # type:np.ndarray
173+
y_transformed = self.target_validator.transform(y_flat)
135174
if y_transformed.ndim == 1:
136175
y_transformed = np.expand_dims(y_transformed, -1)
137176
return np.asarray([]), y_transformed, sequence_lengths
138177

139178
# a matrix that is concatenated by all the time series sequences
140-
X_flat = np.empty([num_data, num_features])
141-
y_flat = np.empty([num_data, num_targets])
179+
180+
X_flat = pd.DataFrame(np.empty([num_data, num_features]), index=group_ids)
181+
y_flat = pd.DataFrame(np.empty([num_data, num_targets]), index=group_ids)
142182

143183
start_idx = 0
144184
for seq_idx, seq_length in enumerate(sequence_lengths):
@@ -152,4 +192,3 @@ def transform(
152192
if y_transformed.ndim == 1:
153193
y_transformed = np.expand_dims(y_transformed, -1)
154194
return X_transformed, y_transformed, sequence_lengths
155-

0 commit comments

Comments
 (0)