Skip to content

enable global settings in parsing topology. #3025

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lcy-seso wants to merge 1 commit into from
Closed

enable global settings in parsing topology. #3025

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 55 additions & 41 deletions python/paddle/trainer_config_helpers/attrs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,23 +58,32 @@ def is_compatible_with(x, Type):

class HookAttribute(object):
"""
Hook Attribute object. As a member of ParameterAttribute class, the hook is an auxiliary operation that occurs
during training process of a layer with parameters, such as img_conv layer, fc layer.

:param type: Hook type, currently supported types:
'pruning' : user specify a sparsity_ratio before training started, and the
network will prune the parameters based on the sparsity_ratio.
eg: The definition of Hook object can be hk = HookAttribute('pruning', 0.6)
The specific usage can be paddle.layer.img_conv(input=img, filter_size=3,
num_channels=3, num_filters=64,
param_attr=ParameterAttribute(update_hooks=hk) )
The pruning details can be found https://arxiv.org/pdf/1506.02626.pdf
:type type: string
Hook Attribute object. As a member of ParameterAttribute class,
the hook is an auxiliary operation that occurs during training process of
a layer with parameters, such as img_conv layer, fc layer.

Reference:
Learning both Weights and Connections for Efficient Neural Networks
https://arxiv.org/pdf/1506.02626.pdf

The example usage is:

.. code-block:: python

:param sparsity_ratio: Must be specified if hook type is 'pruning',
it represents the ratio of the zero elements to be set by the Parameter.
paddle.layer.img_conv(input=img, filter_size=3,
num_channels=3, num_filters=64,
param_attr=ParameterAttribute(update_hooks=hk) )


:param type: Hook type, currently supported types:
'pruning' : user specify a sparsity_ratio before training started, and the
network will prune the parameters based on the sparsity_ratio.
eg: The definition of Hook object can be hk = HookAttribute('pruning', 0.6)
:type type: string
:param sparsity_ratio: Must be specified if hook type is 'pruning',
it represents the ratio of the zero elements to be set by the Parameter.
:type sparsity_ratio: float or None

"""

def __init__(self, type, sparsity_ratio=None):
Expand All @@ -84,7 +93,8 @@ def __init__(self, type, sparsity_ratio=None):
assert is_compatible_with(
self.sparsity_ratio,
float), 'sparisity_ratio must be float type'
assert self.sparsity_ratio <= 1 and self.sparsity_ratio >= 0, 'sparsity_ratio must be a float between [0, 1] '
assert self.sparsity_ratio <= 1 and self.sparsity_ratio >= 0, \
'sparsity_ratio must be a float between [0, 1] '

def __call__(self):
return ParameterHook(self.type, sparsity_ratio=self.sparsity_ratio)
Expand Down Expand Up @@ -139,6 +149,7 @@ class ParameterAttribute(object):
def __init__(self,
name=None,
is_static=False,
initial_smart=None,
initial_std=None,
initial_mean=None,
initial_max=None,
Expand All @@ -152,32 +163,35 @@ def __init__(self,
update_hooks=None,
initializer=None):
self.attr = {}

if is_static:
self.attr['is_static'] = True

if initial_std is None and initial_mean is None and initial_max \
is None and initial_min is None:
self.attr['initial_smart'] = True
elif is_compatible_with(initial_std, float) or \
is_compatible_with(initial_mean, float):
if initial_std is not None:
self.attr['initial_std'] = initial_std
if initial_mean is not None:
self.attr['is_static'] = is_static

if initial_smart is not None:
self.attr['initial_smart'] = initial_smart

if initial_std is not None or initial_mean is not None or \
initial_max is not None or initial_min is not None:
# smart initalization will be ignored, because user customizes
# parameters related to initialization distribution
self.attr['initial_smart'] = False
if is_compatible_with(initial_std, float) or \
is_compatible_with(initial_mean, float):
if initial_std is not None:
self.attr['initial_std'] = initial_std
if initial_mean is not None:
self.attr['initial_mean'] = initial_mean
self.attr['initial_strategy'] = 0 # Gauss Random
elif is_compatible_with(initial_max, float) and \
is_compatible_with(initial_min, float):
initial_max = initial_max
initial_min = initial_min
assert initial_min < initial_max
initial_mean = (initial_max + initial_min) / 2
initial_std = initial_mean - initial_min
self.attr['initial_mean'] = initial_mean
self.attr['initial_strategy'] = 0 # Gauss Random
elif is_compatible_with(initial_max, float) and \
is_compatible_with(initial_min, float):
initial_max = initial_max
initial_min = initial_min
assert initial_min < initial_max
initial_mean = (initial_max + initial_min) / 2
initial_std = initial_mean - initial_min
self.attr['initial_mean'] = initial_mean
self.attr['initial_std'] = initial_std
self.attr['initial_strategy'] = 1 # Uniform Random
else:
raise RuntimeError("Unexpected branch.")
self.attr['initial_std'] = initial_std
self.attr['initial_strategy'] = 1 # Uniform Random
else:
raise RuntimeError("Unexpected branch.")

if not is_static and is_compatible_with(l1_rate, float):
self.attr['decay_rate_l1'] = l1_rate
Expand Down
4 changes: 2 additions & 2 deletions python/paddle/trainer_config_helpers/optimizers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ class MomentumOptimizer(BaseSGDOptimizer):
u_t &= u_{t-1} - \\alpha_t \\gamma_t g_t \\\\
v_t &= v_{t-1} + \\tau_{t-1} \\alpha_t \\gamma_t g_t \\\\
\\tau_t &= \\tau_{t-1} + \\beta_t / \\alpha_t
where :math:`k` is momentum, :math:`\\lambda` is decay rate,

where :math:`k` is momentum, :math:`\\lambda` is decay rate,
:math:`\\gamma_t` is learning rate at the t'th step.

:param sparse: with sparse support or not.
Expand Down
6 changes: 5 additions & 1 deletion python/paddle/trainer_config_helpers/tests/configs/protostr/img_layers.protostr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,10 @@ parameters {
name: "___conv_0__.w0"
size: 65536
initial_mean: 0.0
initial_std: 0.0441941738242
initial_std: 0.01
initial_strategy: 0
initial_smart: false
is_static: false
}
parameters {
name: "___conv_0__.wbias"
Expand All @@ -128,6 +129,7 @@ parameters {
dims: 1
initial_strategy: 0
initial_smart: false
is_static: false
}
parameters {
name: "___batch_norm_0__.w0"
Expand All @@ -136,6 +138,7 @@ parameters {
initial_std: 0.0
initial_strategy: 0
initial_smart: false
is_static: false
}
parameters {
name: "___batch_norm_0__.w1"
Expand Down Expand Up @@ -170,6 +173,7 @@ parameters {
dims: 64
initial_strategy: 0
initial_smart: false
is_static: false
}
input_layer_names: "image"
output_layer_names: "__pool_0__"
Expand Down
6 changes: 5 additions & 1 deletion python/paddle/trainer_config_helpers/tests/configs/protostr/img_trans_layers.protostr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,10 @@ parameters {
name: "___conv_0__.w0"
size: 65536
initial_mean: 0.0
initial_std: 0.0441941738242
initial_std: 0.01
initial_strategy: 0
initial_smart: false
is_static: false
}
parameters {
name: "___conv_0__.wbias"
Expand All @@ -128,6 +129,7 @@ parameters {
dims: 1
initial_strategy: 0
initial_smart: false
is_static: false
}
parameters {
name: "___batch_norm_0__.w0"
Expand All @@ -136,6 +138,7 @@ parameters {
initial_std: 0.0
initial_strategy: 0
initial_smart: false
is_static: false
}
parameters {
name: "___batch_norm_0__.w1"
Expand Down Expand Up @@ -170,6 +173,7 @@ parameters {
dims: 64
initial_strategy: 0
initial_smart: false
is_static: false
}
input_layer_names: "image"
output_layer_names: "__pool_0__"
Expand Down
Loading