From cdc59911784425bcfbf4d5a8df435d793d3ef0cf Mon Sep 17 00:00:00 2001 From: "Javier M. Duarte" Date: Sun, 7 Nov 2021 14:34:23 -0800 Subject: [PATCH 1/4] fix 2 reshape issues: don't reshape streams for flatten and remove final reshape --- hls4ml/model/optimizer/__init__.py | 3 ++- hls4ml/model/optimizer/passes/repack_stream.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/hls4ml/model/optimizer/__init__.py b/hls4ml/model/optimizer/__init__.py index 04bd24b35a..5012228725 100644 --- a/hls4ml/model/optimizer/__init__.py +++ b/hls4ml/model/optimizer/__init__.py @@ -12,7 +12,7 @@ from hls4ml.model.optimizer.passes.conv_same_pad import InsertZeroPaddingBeforeConv2D from hls4ml.model.optimizer.passes.pointwise import OptimizePointwiseConv from hls4ml.model.optimizer.passes.clone import CloneOutput -from hls4ml.model.optimizer.passes.repack_stream import ReshapeStream, BroadcastStream +from hls4ml.model.optimizer.passes.repack_stream import ReshapeStream, BroadcastStream, RemoveFinalReshape from hls4ml.model.optimizer.passes.transpose_opt import RemoveUselessTranspose from hls4ml.model.optimizer.passes.multi_dense import ReplaceMultidimensionalDenseWithConv @@ -41,6 +41,7 @@ register_pass('optimize_pointwise_conv', OptimizePointwiseConv) register_pass('clone_output', CloneOutput) register_pass('reshape_stream', ReshapeStream) +register_pass('remove_final_reshape', RemoveFinalReshape) register_pass('remove_useless_transpose', RemoveUselessTranspose) register_pass('replace_multidense_conv', ReplaceMultidimensionalDenseWithConv) register_pass('broadcast_stream', BroadcastStream) diff --git a/hls4ml/model/optimizer/passes/repack_stream.py b/hls4ml/model/optimizer/passes/repack_stream.py index f54a264c9d..3eb62a1ee1 100644 --- a/hls4ml/model/optimizer/passes/repack_stream.py +++ b/hls4ml/model/optimizer/passes/repack_stream.py @@ -71,7 +71,8 @@ def config_cpp(self): class ReshapeStream(OptimizerPass): ''' Repacks stream for Reshape layer ''' def match(self, node): - return node.__class__.__name__ == 'Reshape' + # do not run optimizer pass for a flatten layer (1 output dimension) + return node.__class__.__name__ == 'Reshape' and len(node.get_output_variable().shape) > 1 def transform(self, model, node): if model.config.backend.name not in ['Vivado', 'VivadoAccelerator'] or \ @@ -121,3 +122,14 @@ def transform(self, model, node): node.inputs[idx] = brdcst_out return True + +class RemoveFinalReshape(OptimizerPass): + ''' Remove reshape if final layer ''' + def match(self, node): + # match if reshape is final node + return node.__class__.__name__ == 'Reshape' and not node.get_output_nodes() + + def transform(self, model, node): + # remove, but don't rewire because it's the output layer + model.remove_node(node, rewire=False) + return True From 2e7f579e6570bf947240d756a168e434703fdabd Mon Sep 17 00:00:00 2001 From: Sioni Summers Date: Mon, 8 Nov 2021 13:07:31 +0100 Subject: [PATCH 2/4] Add a test for a model with Reshape as the final layer --- test/pytest/test_graph.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/pytest/test_graph.py b/test/pytest/test_graph.py index b25d104a2d..8f9e62f4b8 100644 --- a/test/pytest/test_graph.py +++ b/test/pytest/test_graph.py @@ -1,6 +1,7 @@ import hls4ml import numpy as np import pytest +import tensorflow as tf class Reader: def get_weights_data(self, name, var): @@ -107,3 +108,34 @@ def test_graph_branch(iotype, batch): y = model.predict([X0, X1]).reshape(y_expected.shape) # check the output np.testing.assert_allclose(y, y_expected, rtol=1, atol=2**-16) + +@pytest.mark.parametrize('iotype', ['io_parallel', 'io_stream']) +def test_final_reshape(iotype): + ''' Test case for a model with a Reshape as the final layer ''' + inputs = tf.keras.layers.Input(shape=(1,1,1)) # 1 input pixel + conv = tf.keras.layers.Conv2D(6,1) # 6 filters, 1x1 kernel + x = conv(inputs) + conv.set_weights([np.linspace(1,6,6).reshape(1,1,1,6), np.zeros(6)]) # ascending int weights, 0 bias + x = tf.keras.layers.Reshape((3,2))(x) # reshape the (1,1,6) output to (3,2) + model = tf.keras.models.Model(inputs=inputs, outputs=x) + + # create the HLSModel + config = hls4ml.utils.config_from_keras_model(model, granularity='model') + hls_model = hls4ml.converters.convert_from_keras_model(model, + output_dir=f'hls4mlprj_graph_final_reshape_{iotype}', + backend='Vivado', + io_type = iotype, + hls_config=config) + hls_model.compile() + + # Test on ascending integers. The weights mean that each output pixel/neuron has + # a different value + X = np.linspace(-4,4,9).reshape(9,1,1,1) + y = model.predict(X) + y_hls = hls_model.predict(X).reshape(y.shape) + # because of integer inputs and integer weights, we can expect exact matching + np.testing.assert_allclose(y, y_hls, rtol=0) + + + + From 44bfacdb6201e7dec4a32c35c19e4bbe8ab83e99 Mon Sep 17 00:00:00 2001 From: "Javier M. Duarte" Date: Mon, 8 Nov 2021 07:00:45 -0800 Subject: [PATCH 3/4] swap --- hls4ml/model/optimizer/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls4ml/model/optimizer/__init__.py b/hls4ml/model/optimizer/__init__.py index 5012228725..19915b553e 100644 --- a/hls4ml/model/optimizer/__init__.py +++ b/hls4ml/model/optimizer/__init__.py @@ -40,8 +40,8 @@ register_pass('conv2d_same_pad', InsertZeroPaddingBeforeConv2D) register_pass('optimize_pointwise_conv', OptimizePointwiseConv) register_pass('clone_output', CloneOutput) -register_pass('reshape_stream', ReshapeStream) register_pass('remove_final_reshape', RemoveFinalReshape) +register_pass('reshape_stream', ReshapeStream) register_pass('remove_useless_transpose', RemoveUselessTranspose) register_pass('replace_multidense_conv', ReplaceMultidimensionalDenseWithConv) register_pass('broadcast_stream', BroadcastStream) From 7929d045af1d4f7b503ecda08d6c570e80d79a8e Mon Sep 17 00:00:00 2001 From: "Javier M. Duarte" Date: Mon, 8 Nov 2021 07:01:22 -0800 Subject: [PATCH 4/4] only remove for io_parallel; warn for both io_parallel and io_stream --- hls4ml/model/optimizer/passes/repack_stream.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hls4ml/model/optimizer/passes/repack_stream.py b/hls4ml/model/optimizer/passes/repack_stream.py index 3eb62a1ee1..de2e298419 100644 --- a/hls4ml/model/optimizer/passes/repack_stream.py +++ b/hls4ml/model/optimizer/passes/repack_stream.py @@ -130,6 +130,11 @@ def match(self, node): return node.__class__.__name__ == 'Reshape' and not node.get_output_nodes() def transform(self, model, node): - # remove, but don't rewire because it's the output layer - model.remove_node(node, rewire=False) - return True + if model.config.get_config_value('IOType') == 'io_parallel': + print('WARNING: Final layer is a Reshape, which does not affect the output for io_parallel; removing it') + # remove, but don't rewire because it's the output layer + model.remove_node(node, rewire=False) + return True + elif model.config.get_config_value('IOType') == 'io_stream': + print('WARNING: Final layer is a Reshape, which may incur a large resource cost for io_stream; consider removing it') + return False