Skip to content

Correctly expand dims of pointwise layer #715

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 6, 2023
Merged

Correctly expand dims of pointwise layer #715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d14426f
Correctly expand dims of pointwise layer
vloncar Feb 15, 2023
62bb746
Style rules applied
vloncar Feb 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 38 additions & 15 deletions hls4ml/backends/quartus/passes/pointwise.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,56 @@
import numpy as np
from copy import copy

from hls4ml.model.optimizer import OptimizerPass
from hls4ml.model.layers import register_layer
import numpy as np

from hls4ml.backends.fpga.fpga_layers import PointwiseConv1D, PointwiseConv2D
from hls4ml.backends.quartus.passes.convolution_templates import Conv1DConfigTemplate, Conv1DFunctionTemplate, Conv2DConfigTemplate, Conv2DFunctionTemplate, conv1d_config_template, conv2d_config_template, conv_mult_config_template
from hls4ml.backends.quartus.passes.convolution_templates import (
Conv1DConfigTemplate,
Conv1DFunctionTemplate,
Conv2DConfigTemplate,
Conv2DFunctionTemplate,
conv1d_config_template,
conv2d_config_template,
conv_mult_config_template,
)
from hls4ml.model.layers import register_layer
from hls4ml.model.optimizer import OptimizerPass

'''
Custom hls4ml layer implementation for 1x1 Conv filters using im2col
Allows lower latency andresource usage, due to less loop invocations
'''

pointwise_conv1d_function_template = 'nnet::pointwise_conv_1d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
pointwise_conv2d_function_template = 'nnet::pointwise_conv_2d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
pointwise_conv1d_function_template = (
'nnet::pointwise_conv_1d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
)
pointwise_conv2d_function_template = (
'nnet::pointwise_conv_2d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
)

sepconv1d_include_list = ['nnet_utils/nnet_conv1d.h']
sepconv2d_include_list = ['nnet_utils/nnet_conv2d.h']


class PointwiseConv1DConfigTemplate(Conv1DConfigTemplate):
def __init__(self):
super(Conv1DConfigTemplate, self).__init__(PointwiseConv1D)
self.template = conv1d_config_template
self.mult_template = conv_mult_config_template


class PointwiseConv1DFunctionTemplate(Conv1DFunctionTemplate):
def __init__(self):
super(Conv1DFunctionTemplate, self).__init__(PointwiseConv1D, include_header=sepconv1d_include_list)
self.template = pointwise_conv1d_function_template


class PointwiseConv2DConfigTemplate(Conv2DConfigTemplate):
def __init__(self):
super(Conv2DConfigTemplate, self).__init__(PointwiseConv2D)
self.template = conv2d_config_template
self.mult_template = conv_mult_config_template


class PointwiseConv2DFunctionTemplate(Conv2DFunctionTemplate):
def __init__(self):
super(Conv2DFunctionTemplate, self).__init__(PointwiseConv2D, include_header=sepconv2d_include_list)
Expand All @@ -54,19 +71,25 @@ def register_pointwise(backend):
backend.register_template(PointwiseConv2DConfigTemplate)
backend.register_template(PointwiseConv2DFunctionTemplate)


class OptimizePointwiseConv(OptimizerPass):
def match(self, node):
return node.class_name in ('Conv1D', 'Conv2D') and \
node.get_attr('filt_height', 1) == 1 and \
node.get_attr('filt_width') == 1 and \
node.model.config.get_config_value('IOType') == 'io_parallel'
return (
node.class_name in ('Conv1D', 'Conv2D')
and node.get_attr('filt_height', 1) == 1
and node.get_attr('filt_width') == 1
and node.model.config.get_config_value('IOType') == 'io_parallel'
)

def transform(self, model, node):
dim = node.__class__.__name__[-2:] # '1D' or '2D'
pw_node = model.make_node('PointwiseConv' + dim, node.name, copy(node.attributes), node.inputs.copy(), outputs=node.outputs.copy())
if len(node.weights['weight'].data.shape) == 2: # This can happen if we assign weights of Dense layer to 1x1 Conv2D
pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=(0,1))
dim = node.__class__.__name__[-2:] # '1D' or '2D'
pw_node = model.make_node(
'PointwiseConv' + dim, node.name, copy(node.attributes), node.inputs.copy(), outputs=node.outputs.copy()
)
if len(node.weights['weight'].data.shape) == 2: # This can happen if we assign weights of Dense layer to 1x1 Conv2D
expand_axis = tuple(range(int(dim[0])))
pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=expand_axis)
pw_node.weights['bias'].data = node.weights['bias'].data
model.replace_node(node, pw_node)

return True
49 changes: 35 additions & 14 deletions hls4ml/backends/vivado/passes/pointwise.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,34 +1,51 @@
import numpy as np
from copy import copy

from hls4ml.model.optimizer import OptimizerPass
from hls4ml.model.layers import register_layer
import numpy as np

from hls4ml.backends.fpga.fpga_layers import PointwiseConv1D, PointwiseConv2D
from hls4ml.backends.vivado.passes.convolution_templates import Conv1DConfigTemplate, Conv1DFunctionTemplate, Conv2DConfigTemplate, Conv2DFunctionTemplate, conv1d_config_template, conv2d_config_template, conv_mult_config_template
from hls4ml.backends.vivado.passes.convolution_templates import (
Conv1DConfigTemplate,
Conv1DFunctionTemplate,
Conv2DConfigTemplate,
Conv2DFunctionTemplate,
conv1d_config_template,
conv2d_config_template,
conv_mult_config_template,
)
from hls4ml.model.layers import register_layer
from hls4ml.model.optimizer import OptimizerPass

pointwise_conv1d_function_template = 'nnet::pointwise_conv_1d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
pointwise_conv2d_function_template = 'nnet::pointwise_conv_2d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
pointwise_conv1d_function_template = (
'nnet::pointwise_conv_1d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
)
pointwise_conv2d_function_template = (
'nnet::pointwise_conv_2d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
)

sepconv1d_include_list = ['nnet_utils/nnet_conv1d.h', 'nnet_utils/nnet_sepconv1d_stream.h']
sepconv2d_include_list = ['nnet_utils/nnet_conv2d.h', 'nnet_utils/nnet_sepconv2d_stream.h']


class PointwiseConv1DConfigTemplate(Conv1DConfigTemplate):
def __init__(self):
super(Conv1DConfigTemplate, self).__init__(PointwiseConv1D)
self.template = conv1d_config_template
self.mult_template = conv_mult_config_template


class PointwiseConv1DFunctionTemplate(Conv1DFunctionTemplate):
def __init__(self):
super(Conv1DFunctionTemplate, self).__init__(PointwiseConv1D, include_header=sepconv1d_include_list)
self.template = pointwise_conv1d_function_template


class PointwiseConv2DConfigTemplate(Conv2DConfigTemplate):
def __init__(self):
super(Conv2DConfigTemplate, self).__init__(PointwiseConv2D)
self.template = conv2d_config_template
self.mult_template = conv_mult_config_template


class PointwiseConv2DFunctionTemplate(Conv2DFunctionTemplate):
def __init__(self):
super(Conv2DFunctionTemplate, self).__init__(PointwiseConv2D, include_header=sepconv2d_include_list)
Expand All @@ -49,18 +66,22 @@ def register_pointwise(backend):
backend.register_template(PointwiseConv2DConfigTemplate)
backend.register_template(PointwiseConv2DFunctionTemplate)


class OptimizePointwiseConv(OptimizerPass):
def match(self, node):
return node.class_name in ('Conv1D', 'Conv2D') and \
node.get_attr('filt_height', 1) == 1 and \
node.get_attr('filt_width') == 1
return (
node.class_name in ('Conv1D', 'Conv2D')
and node.get_attr('filt_height', 1) == 1
and node.get_attr('filt_width') == 1
)

def transform(self, model, node):
dim = node.__class__.__name__[-2:] # '1D' or '2D'
dim = node.__class__.__name__[-2:] # '1D' or '2D'
pw_node = model.make_node('PointwiseConv' + dim, node.name, copy(node.attributes), node.inputs.copy())
if len(node.weights['weight'].data.shape) == 2: # This can happen if we assign weights of Dense layer to 1x1 Conv2D
pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=(0,1))
if len(node.weights['weight'].data.shape) == 2: # This can happen if we assign weights of Dense layer to 1x1 Conv2D
expand_axis = tuple(range(int(dim[0])))
pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=expand_axis)
pw_node.weights['bias'].data = node.weights['bias'].data
model.replace_node(node, pw_node)
return True

return True
74 changes: 41 additions & 33 deletions hls4ml/model/optimizer/passes/multi_dense.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
from hls4ml.model.optimizer import OptimizerPass
from hls4ml.model.layers import Dense
import numpy as np

from hls4ml.model.layers import Dense
from hls4ml.model.optimizer import OptimizerPass


class ReplaceMultidimensionalDenseWithConv(OptimizerPass):
def match(self, node):
return isinstance(node, Dense) and \
len(node.get_input_variable().shape) - sum(d==1 for d in node.get_input_variable().shape) > 1
# The above sum checks for the number of dimensions in the Dense with size 1
# The subtraction allows the check to only count the number of dimensions with non-1 size
# For example, this prevents matching for a Dense layer with shape (1,N)
return (
isinstance(node, Dense)
and len(node.get_input_variable().shape) - sum(d == 1 for d in node.get_input_variable().shape) > 1
)
# The above sum checks for the number of dimensions in the Dense with size 1
# The subtraction allows the check to only count the number of dimensions with non-1 size
# For example, this prevents matching for a Dense layer with shape (1,N)

def transform(self, model, node):
dim = len(node.get_input_variable().shape) - 1
dim = len(node.get_input_variable().shape) - 1
input_shape = node.get_input_variable().shape

pointwise_attrs = {
Expand All @@ -22,37 +26,41 @@ def transform(self, model, node):
}

if dim == 1:
pointwise_attrs.update({
'in_width': input_shape[0],
'out_width': input_shape[0],
'filt_width': 1,
'stride_width': 1,
'pad_left': 0,
'pad_right': 0,
})
pointwise_attrs.update(
{
'in_width': input_shape[0],
'out_width': input_shape[0],
'filt_width': 1,
'stride_width': 1,
'pad_left': 0,
'pad_right': 0,
}
)
elif dim == 2:
pointwise_attrs.update({
'in_height': input_shape[0],
'in_width': input_shape[1],
'out_height': input_shape[0],
'out_width': input_shape[1],
'filt_height': 1,
'filt_width': 1,
'stride_height': 1,
'stride_width': 1,
'pad_top': 0,
'pad_bottom': 0,
'pad_left': 0,
'pad_right': 0,
})
pointwise_attrs.update(
{
'in_height': input_shape[0],
'in_width': input_shape[1],
'out_height': input_shape[0],
'out_width': input_shape[1],
'filt_height': 1,
'filt_width': 1,
'stride_height': 1,
'stride_width': 1,
'pad_top': 0,
'pad_bottom': 0,
'pad_left': 0,
'pad_right': 0,
}
)
else:
raise Exception('Cannot replace Dense over {dim}D tensor with Conv{dim}D.'.format(dim=dim))

class_name = 'PointwiseConv' + str(dim) + 'D'
pw_node = model.make_node(class_name, node.name, pointwise_attrs, node.inputs.copy())
if len(node.weights['weight'].data.shape) == 2: # This can happen if we assign weights of Dense layer to 1x1 Conv2D
pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=(0,1))
if len(node.weights['weight'].data.shape) == 2: # This can happen if we assign weights of Dense layer to 1x1 Conv2D
pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=tuple(range(dim)))
pw_node.weights['bias'].data = node.weights['bias'].data
model.replace_node(node, pw_node)

return True