Skip to content

[fix] Dropout bug fix #247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jun 10, 2021
Merged

[fix] Dropout bug fix #247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0900210
fix dropout bug
ravinkohli Jun 3, 2021
2e0d35c
fix dropout shape discrepancy
ravinkohli Jun 3, 2021
2d0bc1e
Fix unit test bug
ravinkohli Jun 9, 2021
3bd25e7
Add tests for dropout shape asper comments from fransisco
ravinkohli Jun 10, 2021
6686328
Fix flake
ravinkohli Jun 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> None:
out_features=self.config["num_units_%d" % i],
blocks_per_group=self.config["blocks_per_group_%d" % i],
last_block_index=(i - 1) * self.config["blocks_per_group_%d" % i],
dropout=self.config['use_dropout']
dropout=self.config[f'dropout_{i}'] if self.config['use_dropout'] else None,
)
)

Expand All @@ -52,7 +52,7 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> None:
return backbone

def _add_group(self, in_features: int, out_features: int,
blocks_per_group: int, last_block_index: int, dropout: bool
blocks_per_group: int, last_block_index: int, dropout: Optional[float]
) -> nn.Module:
"""
Adds a group into the main backbone.
Expand Down Expand Up @@ -206,7 +206,7 @@ def __init__(
out_features: int,
blocks_per_group: int,
block_index: int,
dropout: bool,
dropout: Optional[float],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you please add a unit test where you create a pipeline with a config with dropout=0.123, and just check that the dropout of the layer is set to 0.123 to make sure this does not happen again?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure

activation: nn.Module
):
super(ResBlock, self).__init__()
Expand Down
12 changes: 8 additions & 4 deletions autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,14 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> None:
self.config.update(
{"num_units_%d" % (i): num for i, num in enumerate(neuron_counts)}
)
if self.config['use_dropout'] and self.config["max_dropout"] > 0.05:
# we are skipping the last layer, as the function get_shaped_neuron_counts
# is built for getting neuron counts, so it will add the out_features to
# the last layer. However, in dropout we dont want to have that, we just
# want to use the shape and not worry about the output.
if self.config['use_dropout']:
dropout_shape = get_shaped_neuron_counts(
self.config['resnet_shape'], 0, 0, 1000, self.config['num_groups']
)
self.config['resnet_shape'], 0, 0, 1000, self.config['num_groups'] + 1
)[:-1]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi, I immediately do not understand this. The comment is very clear and I get the intention... Can we add a unit test for this?

So that we call build_backbone and we get a model. Then we check that dropout was added as expected?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure


dropout_shape = [
dropout / 1000 * self.config["max_dropout"] for dropout in dropout_shape
Expand All @@ -61,7 +65,7 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> None:
out_features=self.config["num_units_%d" % i],
blocks_per_group=self.config["blocks_per_group"],
last_block_index=(i - 1) * self.config["blocks_per_group"],
dropout=self.config['use_dropout']
dropout=self.config[f'dropout_{i}'] if self.config['use_dropout'] else None
)
)

Expand Down
60 changes: 59 additions & 1 deletion test/test_pipeline/components/setup/test_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@
SchedulerChoice
)
from autoPyTorch.pipeline.components.setup.network_backbone import NetworkBackboneChoice
from autoPyTorch.pipeline.components.setup.network_backbone.ResNetBackbone import ResBlock
from autoPyTorch.pipeline.components.setup.network_backbone.ShapedResNetBackbone import ShapedResNetBackbone
from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent
from autoPyTorch.pipeline.components.setup.network_backbone.utils import get_shaped_neuron_counts
from autoPyTorch.pipeline.components.setup.network_head import NetworkHeadChoice
from autoPyTorch.pipeline.components.setup.network_head.base_network_head import NetworkHeadComponent
from autoPyTorch.pipeline.components.setup.network_initializer import (
Expand All @@ -33,7 +36,10 @@
BaseOptimizerComponent,
OptimizerChoice
)
from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
from autoPyTorch.utils.hyperparameter_search_space_update import (
HyperparameterSearchSpace,
HyperparameterSearchSpaceUpdates
)


class DummyLR(BaseLRComponent):
Expand Down Expand Up @@ -417,6 +423,58 @@ def test_add_network_backbone(self):
# clear addons
base_network_backbone_choice._addons = ThirdPartyComponents(NetworkBackboneComponent)

@pytest.mark.parametrize('resnet_shape', ['funnel', 'long_funnel',
'diamond', 'hexagon',
'brick', 'triangle',
'stairs'])
def test_dropout(self, resnet_shape):
# ensures that dropout is assigned to the resblock as expected
dataset_properties = {"task_type": constants.TASK_TYPES_TO_STRING[1]}
max_dropout = 0.5
num_groups = 4
config_space = ShapedResNetBackbone.get_hyperparameter_search_space(dataset_properties=dataset_properties,
use_dropout=HyperparameterSearchSpace(
hyperparameter='use_dropout',
value_range=[True],
default_value=True),
max_dropout=HyperparameterSearchSpace(
hyperparameter='max_dropout',
value_range=[max_dropout],
default_value=max_dropout),
resnet_shape=HyperparameterSearchSpace(
hyperparameter='resnet_shape',
value_range=[resnet_shape],
default_value=resnet_shape),
num_groups=HyperparameterSearchSpace(
hyperparameter='num_groups',
value_range=[num_groups],
default_value=num_groups),
blocks_per_group=HyperparameterSearchSpace(
hyperparameter='blocks_per_group',
value_range=[1],
default_value=1
)
)

config = config_space.sample_configuration().get_dictionary()
resnet_backbone = ShapedResNetBackbone(**config)
resnet_backbone.build_backbone((100, 5))
dropout_probabilites = [resnet_backbone.config[key] for key in resnet_backbone.config if 'dropout_' in key]
dropout_shape = get_shaped_neuron_counts(
resnet_shape, 0, 0, 1000, num_groups + 1
)[:-1]

dropout_shape = [
dropout / 1000 * max_dropout for dropout in dropout_shape
]
blocks_dropout = []
for block in resnet_backbone.backbone:
if isinstance(block, torch.nn.Sequential):
for inner_block in block:
if isinstance(inner_block, ResBlock):
blocks_dropout.append(inner_block.dropout)
assert dropout_probabilites == dropout_shape == blocks_dropout


class TestNetworkHead:
def test_all_heads_available(self):
Expand Down