Skip to content

Commit 067d17f

Browse files
author
Github Actions
committed
Ravin Kohli: Additional metrics during train (#194)
1 parent 963e5f9 commit 067d17f

File tree

150 files changed

+44288
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

150 files changed

+44288
-0
lines changed

‎refactor_development_regularization_cocktails_backup/.buildinfo

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Sphinx build info version 1
2+
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
3+
config: 28ba0151b4e1d0ebfa351d5c8fb73340
4+
tags: 645f666f9bcd5a90fca523b33c5a78b7

‎refactor_development_regularization_cocktails_backup/_downloads/000ffe6d9d5014b4165debb6cbf446f8/example_tabular_regression.py

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
"""
2+
======================
3+
Tabular Regression
4+
======================
5+
6+
The following example shows how to fit a sample regression model
7+
with AutoPyTorch
8+
"""
9+
import os
10+
import tempfile as tmp
11+
import warnings
12+
13+
import sklearn.datasets
14+
import sklearn.model_selection
15+
16+
os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()
17+
os.environ['OMP_NUM_THREADS'] = '1'
18+
os.environ['OPENBLAS_NUM_THREADS'] = '1'
19+
os.environ['MKL_NUM_THREADS'] = '1'
20+
21+
warnings.simplefilter(action='ignore', category=UserWarning)
22+
warnings.simplefilter(action='ignore', category=FutureWarning)
23+
24+
from autoPyTorch.api.tabular_regression import TabularRegressionTask
25+
26+
27+
if __name__ == '__main__':
28+
29+
############################################################################
30+
# Data Loading
31+
# ============
32+
X, y = sklearn.datasets.fetch_openml(name='boston', return_X_y=True, as_frame=True)
33+
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
34+
X,
35+
y,
36+
random_state=1,
37+
)
38+
39+
# Scale the regression targets to have zero mean and unit variance.
40+
# This is important for Neural Networks since predicting large target values would require very large weights.
41+
# One can later rescale the network predictions like this: y_pred = y_pred_scaled * y_train_std + y_train_mean
42+
y_train_mean = y_train.mean()
43+
y_train_std = y_train.std()
44+
45+
y_train_scaled = (y_train - y_train_mean) / y_train_std
46+
y_test_scaled = (y_test - y_train_mean) / y_train_std
47+
48+
############################################################################
49+
# Build and fit a regressor
50+
# ==========================
51+
api = TabularRegressionTask(
52+
temporary_directory='./tmp/autoPyTorch_example_tmp_02',
53+
output_directory='./tmp/autoPyTorch_example_out_02',
54+
# To maintain logs of the run, set the next two as False
55+
delete_tmp_folder_after_terminate=True,
56+
delete_output_folder_after_terminate=True
57+
)
58+
59+
############################################################################
60+
# Search for an ensemble of machine learning algorithms
61+
# =====================================================
62+
api.search(
63+
X_train=X_train,
64+
y_train=y_train_scaled,
65+
X_test=X_test.copy(),
66+
y_test=y_test_scaled.copy(),
67+
optimize_metric='r2',
68+
total_walltime_limit=300,
69+
func_eval_time_limit_secs=50,
70+
enable_traditional_pipeline=False,
71+
)
72+
73+
############################################################################
74+
# Print the final ensemble performance
75+
# ====================================
76+
print(api.run_history, api.trajectory)
77+
y_pred_scaled = api.predict(X_test)
78+
79+
# Rescale the Neural Network predictions into the original target range
80+
y_pred = y_pred_scaled * y_train_std + y_train_mean
81+
score = api.score(y_pred, y_test)
82+
83+
print(score)
84+
# Print the final ensemble built by AutoPyTorch
85+
print(api.show_models())

‎refactor_development_regularization_cocktails_backup/_downloads/0b35eb5fea161bb1ba2c30e6b7323b44/example_single_configuration.ipynb

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"metadata": {
7+
"collapsed": false
8+
},
9+
"outputs": [],
10+
"source": [
11+
"%matplotlib inline"
12+
]
13+
},
14+
{
15+
"cell_type": "markdown",
16+
"metadata": {},
17+
"source": [
18+
"\n# Fit a single configuration\n*Auto-PyTorch* searches for the best combination of machine learning algorithms\nand their hyper-parameter configuration for a given task.\n\nThis example shows how one can fit one of these pipelines, both, with a user defined\nconfiguration, and a randomly sampled one form the configuration space.\nThe pipelines that Auto-PyTorch fits are compatible with Scikit-Learn API. You can\nget further documentation about Scikit-Learn models here: <https://scikit-learn.org/stable/getting_started.html`>_\n"
19+
]
20+
},
21+
{
22+
"cell_type": "code",
23+
"execution_count": null,
24+
"metadata": {
25+
"collapsed": false
26+
},
27+
"outputs": [],
28+
"source": [
29+
"import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.metrics\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\nfrom autoPyTorch.datasets.resampling_strategy import HoldoutValTypes\n\n\nif __name__ == '__main__':\n ############################################################################\n # Data Loading\n # ============\n\n X, y = sklearn.datasets.fetch_openml('iris', return_X_y=True, as_frame=True)\n X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n X, y, test_size=0.8, random_state=3\n )\n\n ############################################################################\n # Define an estimator\n # ============================\n\n # Search for a good configuration\n estimator = TabularClassificationTask(\n resampling_strategy=HoldoutValTypes.holdout_validation,\n resampling_strategy_args={'val_share': 0.33}\n )\n\n ############################################################################\n # Get a random configuration of the pipeline for current dataset\n # ===============================================================\n\n dataset = estimator.get_dataset(X_train=X_train,\n y_train=y_train,\n X_test=X_test,\n y_test=y_test)\n configuration = estimator.get_search_space(dataset).get_default_configuration()\n\n ###########################################################################\n # Fit the configuration\n # ==================================\n\n pipeline, run_info, run_value, dataset = estimator.fit_pipeline(\n X_train=X_train,\n y_train=y_train,\n dataset_name='kr-vs-kp',\n run_time_limit_secs=100,\n X_test=X_test,\n y_test=y_test,\n disable_file_output=False,\n configuration=configuration,\n )\n\n # This object complies with Scikit-Learn Pipeline API.\n # https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html\n print(pipeline.named_steps)\n\n # The fit_pipeline command also returns a named tuple with the pipeline constraints\n print(run_info)\n\n # The fit_pipeline command also returns a named tuple with train/test performance\n print(run_value)\n\n print(\"Passed Configuration:\", pipeline.config)\n print(\"Network:\", pipeline.named_steps['network'].network)"
30+
]
31+
}
32+
],
33+
"metadata": {
34+
"kernelspec": {
35+
"display_name": "Python 3",
36+
"language": "python",
37+
"name": "python3"
38+
},
39+
"language_info": {
40+
"codemirror_mode": {
41+
"name": "ipython",
42+
"version": 3
43+
},
44+
"file_extension": ".py",
45+
"mimetype": "text/x-python",
46+
"name": "python",
47+
"nbconvert_exporter": "python",
48+
"pygments_lexer": "ipython3",
49+
"version": "3.8.9"
50+
}
51+
},
52+
"nbformat": 4,
53+
"nbformat_minor": 0
54+
}

‎refactor_development_regularization_cocktails_backup/_downloads/2f0f54a9831653ac5c12ee0e06324a62/basics_tabular_jupyter.zip

7.42 KB
Binary file not shown.

‎refactor_development_regularization_cocktails_backup/_downloads/342871cbb8ddcf6157ab171f9b9eab25/example_resampling_strategy.ipynb

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"metadata": {
7+
"collapsed": false
8+
},
9+
"outputs": [],
10+
"source": [
11+
"%matplotlib inline"
12+
]
13+
},
14+
{
15+
"cell_type": "markdown",
16+
"metadata": {},
17+
"source": [
18+
"\n# Tabular Classification with different resampling strategy\n\nThe following example shows how to fit a sample classification model\nwith different resampling strategies in AutoPyTorch\nBy default, AutoPyTorch uses Holdout Validation with\na 67% train size split.\n"
19+
]
20+
},
21+
{
22+
"cell_type": "code",
23+
"execution_count": null,
24+
"metadata": {
25+
"collapsed": false
26+
},
27+
"outputs": [],
28+
"source": [
29+
"import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.model_selection\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\nfrom autoPyTorch.datasets.resampling_strategy import CrossValTypes, HoldoutValTypes\n\n\nif __name__ == '__main__':\n\n ############################################################################\n # Data Loading\n # ============\n X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True)\n X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n X,\n y,\n random_state=1,\n )\n\n ############################################################################\n # Build and fit a classifier with default resampling strategy\n # ===========================================================\n api = TabularClassificationTask(\n temporary_directory='./tmp/autoPyTorch_example_tmp_03',\n output_directory='./tmp/autoPyTorch_example_out_03',\n # To maintain logs of the run, set the next two as False\n delete_tmp_folder_after_terminate=True,\n delete_output_folder_after_terminate=True,\n # 'HoldoutValTypes.holdout_validation' with 'val_share': 0.33\n # is the default argument setting for TabularClassificationTask.\n # It is explicitly specified in this example for demonstrational\n # purpose.\n resampling_strategy=HoldoutValTypes.holdout_validation,\n resampling_strategy_args={'val_share': 0.33}\n )\n\n ############################################################################\n # Search for an ensemble of machine learning algorithms\n # =====================================================\n api.search(\n X_train=X_train,\n y_train=y_train,\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=150,\n func_eval_time_limit_secs=30\n )\n\n ############################################################################\n # Print the final ensemble performance\n # ====================================\n print(api.run_history, api.trajectory)\n y_pred = api.predict(X_test)\n score = api.score(y_pred, y_test)\n print(score)\n # Print the final ensemble built by AutoPyTorch\n print(api.show_models())\n\n ############################################################################\n\n ############################################################################\n # Build and fit a classifier with Cross validation resampling strategy\n # ====================================================================\n api = TabularClassificationTask(\n temporary_directory='./tmp/autoPyTorch_example_tmp_04',\n output_directory='./tmp/autoPyTorch_example_out_04',\n # To maintain logs of the run, set the next two as False\n delete_tmp_folder_after_terminate=True,\n delete_output_folder_after_terminate=True,\n resampling_strategy=CrossValTypes.k_fold_cross_validation,\n resampling_strategy_args={'num_splits': 3}\n )\n\n ############################################################################\n # Search for an ensemble of machine learning algorithms\n # =====================================================\n api.search(\n X_train=X_train,\n y_train=y_train,\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=150,\n func_eval_time_limit_secs=30\n )\n\n ############################################################################\n # Print the final ensemble performance\n # ====================================\n print(api.run_history, api.trajectory)\n y_pred = api.predict(X_test)\n score = api.score(y_pred, y_test)\n print(score)\n # Print the final ensemble built by AutoPyTorch\n print(api.show_models())\n\n ############################################################################\n\n ############################################################################\n # Build and fit a classifier with Stratified resampling strategy\n # ==============================================================\n api = TabularClassificationTask(\n temporary_directory='./tmp/autoPyTorch_example_tmp_05',\n output_directory='./tmp/autoPyTorch_example_out_05',\n # To maintain logs of the run, set the next two as False\n delete_tmp_folder_after_terminate=True,\n delete_output_folder_after_terminate=True,\n # For demonstration purposes, we use\n # Stratified hold out validation. However,\n # one can also use CrossValTypes.stratified_k_fold_cross_validation.\n resampling_strategy=HoldoutValTypes.stratified_holdout_validation,\n resampling_strategy_args={'val_share': 0.33}\n )\n\n ############################################################################\n # Search for an ensemble of machine learning algorithms\n # =====================================================\n api.search(\n X_train=X_train,\n y_train=y_train,\n X_test=X_test.copy(),\n y_test=y_test.copy(),\n optimize_metric='accuracy',\n total_walltime_limit=150,\n func_eval_time_limit_secs=30\n )\n\n ############################################################################\n # Print the final ensemble performance\n # ====================================\n print(api.run_history, api.trajectory)\n y_pred = api.predict(X_test)\n score = api.score(y_pred, y_test)\n print(score)\n # Print the final ensemble built by AutoPyTorch\n print(api.show_models())"
30+
]
31+
}
32+
],
33+
"metadata": {
34+
"kernelspec": {
35+
"display_name": "Python 3",
36+
"language": "python",
37+
"name": "python3"
38+
},
39+
"language_info": {
40+
"codemirror_mode": {
41+
"name": "ipython",
42+
"version": 3
43+
},
44+
"file_extension": ".py",
45+
"mimetype": "text/x-python",
46+
"name": "python",
47+
"nbconvert_exporter": "python",
48+
"pygments_lexer": "ipython3",
49+
"version": "3.8.9"
50+
}
51+
},
52+
"nbformat": 4,
53+
"nbformat_minor": 0
54+
}

0 commit comments

Comments
 (0)