Skip to content

Fix RNN layers when strategy=resource #780

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 8, 2023
Merged

Fix RNN layers when strategy=resource #780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0678d72
Fix RNN transpose and align with rest of the code
vloncar May 3, 2023
e2c3e50
Nicer name of _t types in generated defines.h
vloncar May 3, 2023
51c117b
Use strategy in prj_name to ensure different name
vloncar May 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 3 additions & 21 deletions hls4ml/backends/quartus/quartus_backend.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import numpy as np

from hls4ml.backends import FPGABackend
from hls4ml.model.attributes import ConfigurableAttribute
from hls4ml.model.attributes import ConfigurableAttribute, TypeAttribute
from hls4ml.model.flow import register_flow
from hls4ml.model.layers import GRU, LSTM, Activation, Conv1D, Conv2D, Dense, Embedding, Layer, SimpleRNN, Softmax
from hls4ml.model.optimizer import get_backend_passes, layer_optimizer
Expand Down Expand Up @@ -39,6 +39,8 @@ def _register_layer_attributes(self):
for layer in rnn_layers:
attrs = self.attribute_map.get(layer, [])
attrs.append(ConfigurableAttribute('recurrent_reuse_factor', default=1))
attrs.append(ConfigurableAttribute('table_size', default=1024))
attrs.append(TypeAttribute('table', default=FixedPrecisionType(18, 8)))
self.attribute_map[layer] = attrs

def _register_flows(self):
Expand Down Expand Up @@ -312,16 +314,6 @@ def init_lstm(self, layer):
reuse_factor = layer.model.config.get_reuse_factor(layer)
layer.set_attr('recurrent_reuse_factor', reuse_factor)

index_t = IntegerPrecisionType(width=1, signed=False)
layer.set_attr('index_t', index_t)

if 'table_t' not in layer.attributes:
layer.set_attr(
'table_t', NamedType(name=layer.name + '_table_t', precision=FixedPrecisionType(width=18, integer=8))
)
if 'table_size' not in layer.attributes:
layer.set_attr('table_size', 1024)

# We don't use RF yet
if True: # layer.model.config.is_resource_strategy(layer): ... Quartus only supports Dense resource multiplication
n_in, n_out, n_in_recr, n_out_recr = self.get_layer_mult_size(layer)
Expand Down Expand Up @@ -367,14 +359,4 @@ def init_simple_rnn(self, layer):
reuse_factor = layer.model.config.get_reuse_factor(layer)
layer.set_attr('recurrent_reuse_factor', reuse_factor)

index_t = IntegerPrecisionType(width=1, signed=False)
layer.set_attr('index_t', index_t)

if 'table_t' not in layer.attributes:
layer.set_attr(
'table_t', NamedType(name=layer.name + '_table_t', precision=FixedPrecisionType(width=18, integer=8))
)
if 'table_size' not in layer.attributes:
layer.set_attr('table_size', 1024)

# TODO - Consider setting and using RF
6 changes: 3 additions & 3 deletions hls4ml/backends/vivado/passes/recurrent_templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
typedef {accum_t.name} accum_t;
typedef {bias_t.name} bias_t;
typedef {weight_t.name} weight_t;
typedef ap_{index_t} index_t;
typedef {index_t.name} index_t;
template<class x_T, class y_T>
using product = nnet::product::{product_type}<x_T, y_T>;
}};\n"""
Expand All @@ -28,15 +28,15 @@
static const unsigned table_size = {table_size};
static const unsigned io_type = nnet::{iotype};
static const unsigned reuse_factor = {reuse};
typedef ap_{table_t} table_t;
typedef {table_t.name} table_t;
}};\n"""

recr_activ_config_template = """struct {type}_config{index}_recr : nnet::activ_config {{
static const unsigned n_in = {n_in};
static const unsigned table_size = {table_size};
static const unsigned io_type = nnet::{iotype};
static const unsigned reuse_factor = {reuse};
typedef ap_{table_t} table_t;
typedef {table_t.name} table_t;
}};\n"""

# LSTM + GRU templates
Expand Down
24 changes: 5 additions & 19 deletions hls4ml/backends/vivado/vivado_backend.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from hls4ml.backends import FPGABackend
from hls4ml.backends.fpga.fpga_types import APTypeConverter, HLSTypeConverter, VivadoArrayVariableConverter
from hls4ml.model.attributes import ChoiceAttribute, ConfigurableAttribute
from hls4ml.model.attributes import ChoiceAttribute, ConfigurableAttribute, TypeAttribute
from hls4ml.model.flow import register_flow
from hls4ml.model.layers import (
GRU,
Expand Down Expand Up @@ -51,6 +51,8 @@ def _register_layer_attributes(self):
attrs = self.attribute_map.get(layer, [])
attrs.append(ConfigurableAttribute('recurrent_reuse_factor', default=1))
attrs.append(ConfigurableAttribute('static', value_type=bool, default=True))
attrs.append(ConfigurableAttribute('table_size', default=1024))
attrs.append(TypeAttribute('table', default=FixedPrecisionType(18, 8)))
self.attribute_map[layer] = attrs

# Add ParallelizationFactor to Conv1D/2D
Expand Down Expand Up @@ -393,46 +395,30 @@ def init_lstm(self, layer):
reuse_factor = layer.model.config.get_reuse_factor(layer)
layer.set_attr('recurrent_reuse_factor', reuse_factor)

index_t = IntegerPrecisionType(width=1, signed=False)

if 'table_t' not in layer.attributes:
layer.set_attr('table_t', FixedPrecisionType(width=18, integer=8))
if 'table_size' not in layer.attributes:
layer.set_attr('table_size', 1024)
if layer.model.config.is_resource_strategy(layer):
n_in, n_out, n_in_recr, n_out_recr = self.get_layer_mult_size(layer)
self.set_closest_reuse_factor(layer, n_in, n_out)
self.set_closest_reuse_factor(layer, n_in_recr, n_out_recr, attribute='recurrent_reuse_factor')
layer.weights['weight'].data = np.transpose(layer.weights['weight'].data)
layer.weights['recurrent_weight'].data = np.transpose(layer.weights['recurrent_weight'].data)
layer.set_attr('strategy', 'resource')
else:
layer.set_attr('strategy', 'latency')

layer.set_attr('index_t', index_t)
layer.set_attr('index_t', NamedType(f'layer{layer.index}_index', IntegerPrecisionType(width=1, signed=False)))

@layer_optimizer(GRU)
def init_gru(self, layer):
reuse_factor = layer.model.config.get_reuse_factor(layer)
layer.set_attr('recurrent_reuse_factor', reuse_factor)

index_t = IntegerPrecisionType(width=1, signed=False)

if 'table_t' not in layer.attributes:
layer.set_attr('table_t', FixedPrecisionType(width=18, integer=8))
if 'table_size' not in layer.attributes:
layer.set_attr('table_size', 1024)
if layer.model.config.is_resource_strategy(layer):
n_in, n_out, n_in_recr, n_out_recr = self.get_layer_mult_size(layer)
self.set_closest_reuse_factor(layer, n_in, n_out)
self.set_closest_reuse_factor(layer, n_in_recr, n_out_recr, attribute='recurrent_reuse_factor')
layer.weights['weight'].data = np.transpose(layer.weights['weight'].data)
layer.weights['recurrent_weight'].data = np.transpose(layer.weights['recurrent_weight'].data)
layer.set_attr('strategy', 'resource')
else:
layer.set_attr('strategy', 'latency')

layer.set_attr('index_t', index_t)
layer.set_attr('index_t', NamedType(f'layer{layer.index}_index', IntegerPrecisionType(width=1, signed=False)))

@layer_optimizer(GarNet)
def init_garnet(self, layer):
Expand Down
2 changes: 1 addition & 1 deletion hls4ml/model/layers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def __init__(self, model, name, attributes, inputs, outputs=None):
config_value, str
): # TODO maybe move this to __setitem__ of AttributeDict?
precision = self.model.config.backend.convert_precision_string(config_value)
config_value = NamedType(self.name + config_key, precision)
config_value = NamedType(self.name + '_' + config_key, precision)
self.attributes[config_key] = config_value

self.initialize()
Expand Down
8 changes: 5 additions & 3 deletions test/pytest/test_rnn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ def test_rnn_parsing(rnn_layer, return_sequences):
)
@pytest.mark.parametrize('return_sequences', [True, False])
@pytest.mark.parametrize('static', [True, False])
def test_rnn_accuracy(rnn_layer, return_sequences, backend, io_type, static):
@pytest.mark.parametrize('strategy', ['latency', 'resource'])
def test_rnn_accuracy(rnn_layer, return_sequences, backend, io_type, strategy, static):
# Subtract 0.5 to include negative values
input_shape = (12, 8)
X = np.random.rand(50, *input_shape) - 0.5
Expand All @@ -109,8 +110,9 @@ def test_rnn_accuracy(rnn_layer, return_sequences, backend, io_type, static):
keras_model, granularity='name', default_precision=default_precision, backend=backend
)
hls_config['LayerName'][layer_name]['static'] = static
prj_name = 'hls4mlprj_rnn_accuracy_{}_static_{}_ret_seq_{}_{}_{}'.format(
rnn_layer.__class__.__name__.lower(), int(static), int(return_sequences), backend, io_type
hls_config['LayerName'][layer_name]['Strategy'] = strategy
prj_name = 'hls4mlprj_rnn_accuracy_{}_static_{}_ret_seq_{}_{}_{}_{}'.format(
rnn_layer.__class__.__name__.lower(), int(static), int(return_sequences), backend, io_type, strategy
)
output_dir = str(test_root_path / prj_name)

Expand Down