Skip to content

Commit 64694f2

Browse files
vloncarthesps
andauthored
Update flow dependencies (#588)
* Shuffle the flows a bit - Make writer flows depend on the ip flows - Add reapply policy to teh ModelGraph.apply_flows() - Move the stamp to the optimizer * Remove the printout * Add a test for flows Co-authored-by: Sioni Summers <[email protected]>
1 parent 27b22b0 commit 64694f2

File tree

8 files changed

+160
-20
lines changed

8 files changed

+160
-20
lines changed

hls4ml/backends/quartus/quartus_backend.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ def _register_flows(self):
5151
template_flow = register_flow('apply_templates', templates, requires=[init_flow], backend=self.name)
5252

5353
writer_passes = [
54+
'make_stamp',
5455
'quartus:write_hls'
5556
]
5657
writer_flow_requirements = ['optimize', quartus_types_flow, template_flow]

hls4ml/backends/vivado/vivado_backend.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,10 @@ def _register_flows(self):
6767
template_flow = register_flow('apply_templates', self._get_layer_templates, requires=[init_flow], backend=self.name)
6868

6969
writer_passes = [
70+
'make_stamp',
7071
'vivado:write_hls'
7172
]
72-
writer_flow_requirements = ['optimize', vivado_types_flow, template_flow]
73-
self._writer_flow = register_flow('write', writer_passes, requires=writer_flow_requirements, backend=self.name)
73+
self._writer_flow = register_flow('write', writer_passes, requires=['vivado:ip'], backend=self.name)
7474

7575
all_passes = get_backend_passes(self.name)
7676

hls4ml/backends/vivado_accelerator/vivado_accelerator_backend.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ def create_initial_config(self, board='pynq-z2', part=None, clock_period=5, io_t
9999
return config
100100

101101
def _register_flows(self):
102-
vivado_writer = ['vivado:write']
103-
vivado_accel_writer = ['vivadoaccelerator:write_hls']
104-
self._writer_flow = register_flow('write', vivado_accel_writer, requires=vivado_writer, backend=self.name)
105-
self._default_flow = 'vivado:ip'
102+
vivado_ip = 'vivado:ip'
103+
writer_passes = ['make_stamp', 'vivadoaccelerator:write_hls']
104+
self._writer_flow = register_flow('write', writer_passes, requires=[vivado_ip], backend=self.name)
105+
self._default_flow = vivado_ip

hls4ml/model/graph.py

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -333,8 +333,37 @@ def _make_graph(self, layer_list):
333333

334334
self.graph[name] = self.make_node(kind, name, layer, inputs, outputs)
335335

336-
def apply_flow(self, flow):
337-
applied_flows = {}
336+
def apply_flow(self, flow, reapply='single'):
337+
"""Applies a flow (a collection of optimizers).
338+
Args:
339+
flow (str): The name of the flow to apply
340+
reapply (str, optional): Determines the action to take if the flow and its requirements have already been
341+
applied. Possible values are:
342+
- 'all': Apply the flow and all its requirements.
343+
- 'single': Apply only the given flow, but skip the already applied requirements.
344+
- 'none': Skip applying the flow.
345+
Defaults to 'single'.
346+
"""
347+
def all_applied_flows():
348+
applied_flows = {}
349+
350+
for flow_group in self._applied_flows:
351+
applied_flows.update({flow: [] for flow in flow_group.keys()})
352+
353+
return applied_flows
354+
355+
assert reapply in ['all', 'single', 'none']
356+
357+
if reapply == 'all':
358+
applied_flows = {}
359+
elif reapply == 'single':
360+
applied_flows = all_applied_flows()
361+
applied_flows.pop(flow, None)
362+
else: # reapply == 'none'
363+
applied_flows = all_applied_flows()
364+
if flow in applied_flows:
365+
return
366+
338367
self._apply_sub_flow(flow, applied_flows)
339368
self._applied_flows.append(applied_flows)
340369

@@ -552,16 +581,6 @@ def write(self):
552581
directory specified in the `config`.
553582
"""
554583

555-
def _make_stamp():
556-
""" Create a unique identifier for the generated code. This identifier is used to
557-
compile a unique library and link it with python. """
558-
from string import hexdigits
559-
from random import choice
560-
length = 8
561-
return ''.join(choice(hexdigits) for m in range(length))
562-
563-
self.config.config['Stamp'] = _make_stamp()
564-
565584
self.config.backend.write(self)
566585

567586
def compile(self):

hls4ml/model/optimizer/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from hls4ml.model.flow.flow import register_flow
22
import os
33

4-
from hls4ml.model.optimizer.optimizer import OptimizerPass, GlobalOptimizerPass, LayerOptimizerPass, ConfigurableOptimizerPass, register_pass, get_optimizer, optimize_model, get_available_passes, get_backend_passes, optimizer_pass, layer_optimizer, model_optimizer, extract_optimizers_from_path, extract_optimizers_from_object
4+
from hls4ml.model.optimizer.optimizer import OptimizerPass, GlobalOptimizerPass, LayerOptimizerPass, ModelOptimizerPass, ConfigurableOptimizerPass, register_pass, get_optimizer, optimize_model, get_available_passes, get_backend_passes, optimizer_pass, layer_optimizer, model_optimizer, extract_optimizers_from_path, extract_optimizers_from_object
55

66

77
opt_path = os.path.dirname(__file__) + '/passes'
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from hls4ml.model.optimizer import ModelOptimizerPass
2+
3+
4+
class MakeStamp(ModelOptimizerPass):
5+
def __init__(self):
6+
self.name = 'make_stamp'
7+
8+
def transform(self, model):
9+
def _make_stamp():
10+
""" Create a unique identifier for the generated code. This identifier is used to
11+
compile a unique library and link it with python. """
12+
from string import hexdigits
13+
from random import choice
14+
length = 8
15+
return ''.join(choice(hexdigits) for m in range(length))
16+
17+
model.config.config['Stamp'] = _make_stamp()
18+
19+
return False # No model graph changes made

test/pytest/test_flows.py

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import hls4ml
2+
import pytest
3+
4+
'''
5+
Tests for model flows.
6+
Construct some dummy optimizer passes and flows that do nothing.
7+
Passes record their label to the model.
8+
Tests check that the order of applied passes matches the expectations
9+
'''
10+
11+
class DummyPass(hls4ml.model.optimizer.OptimizerPass):
12+
def __init__(self, label):
13+
self.label = label
14+
def match(self, node):
15+
return True
16+
def transform(self, model, node):
17+
if getattr(model, 'test_flow_passes', None) is None:
18+
model.test_flow_passes = []
19+
model.test_flow_passes.append(self.label)
20+
return False
21+
22+
class DummyPassA(DummyPass):
23+
def __init__(self):
24+
super().__init__('A')
25+
class DummyPassB(DummyPass):
26+
def __init__(self):
27+
super().__init__('B')
28+
class DummyPassC(DummyPass):
29+
def __init__(self):
30+
super().__init__('C')
31+
32+
hls4ml.model.optimizer.register_pass('A', DummyPassA)
33+
hls4ml.model.optimizer.register_pass('B', DummyPassB)
34+
hls4ml.model.optimizer.register_pass('C', DummyPassC)
35+
36+
DummyFlowA = hls4ml.model.flow.register_flow('A', ['A'])
37+
DummyFlowB = hls4ml.model.flow.register_flow('B', ['B'])
38+
DummyFlowC = hls4ml.model.flow.register_flow('C', ['C'])
39+
DummyFlowAB = hls4ml.model.flow.register_flow('AB', ['A', 'B'])
40+
DummyFlowBReqA = hls4ml.model.flow.register_flow('BReqA', ['B'], requires=[DummyFlowA])
41+
DummyFlowCReqBReqA = hls4ml.model.flow.register_flow('CReqBReqA', ['C'], requires=[DummyFlowBReqA])
42+
43+
def dummy_flow_model():
44+
layers = [{'class_name' : 'Input', 'name' : 'layer0_input', 'input_shape' : [1]}]
45+
config = {'HLSConfig':{'Model':{'Precision':'ap_fixed<32,16>','ReuseFactor' : 1},
46+
'Flows': []}}
47+
model = hls4ml.model.ModelGraph(config, None, layers)
48+
return model
49+
50+
class FlowTester:
51+
index = 0
52+
def __init__(self, flows_to_apply, expected_pass_order, reapply):
53+
self.flows_to_apply = flows_to_apply
54+
self.expected_pass_order = expected_pass_order
55+
self.reapply = reapply
56+
self.index = FlowTester.index
57+
FlowTester.index += 1
58+
def run(self):
59+
model = dummy_flow_model()
60+
model.test_flow_passes = []
61+
for flow in self.flows_to_apply:
62+
model.apply_flow(flow, self.reapply)
63+
self.observed_pass_order = model.test_flow_passes
64+
return self.observed_pass_order == self.expected_pass_order
65+
66+
flow_tests = [FlowTester(['A', 'B', 'C'], ['A', 'B', 'C'], 'single'), # independent flows in order
67+
FlowTester(['A', 'A'], ['A', 'A'], 'single'), # same flow twice, single application
68+
FlowTester(['A', 'A'], ['A', 'A'], 'all'), # same flow twice with reapply
69+
FlowTester(['A', 'A'], ['A'], 'none'), # same flow twice with none
70+
FlowTester(['BReqA'], ['A', 'B'], 'single'), # one flow with a dependency
71+
FlowTester(['CReqBReqA'], ['A', 'B', 'C'], 'single'), # one flow with dependency chain
72+
FlowTester(['CReqBReqA', 'A'], ['A', 'B', 'C', 'A'], 'single'), # one flow with dependency chain, repeat dependency
73+
FlowTester(['CReqBReqA', 'A'], ['A', 'B', 'C', 'A'], 'all'), # one flow with dependency chain, repeat dependency
74+
FlowTester(['CReqBReqA', 'A'], ['A', 'B', 'C'], 'none'), # one flow with dependency chain, repeat depencency
75+
FlowTester(['A', 'CReqBReqA'], ['A', 'B', 'C'], 'single'), # one flow with dependency chain, repeat depencency
76+
FlowTester(['A', 'CReqBReqA'], ['A', 'A', 'B', 'C'], 'all'), # one flow with dependency chain, repeat depencency
77+
FlowTester(['A', 'CReqBReqA'], ['A', 'B', 'C'], 'none'), # one flow with dependency chain, repeat depencency
78+
FlowTester(['A', 'BReqA'], ['A', 'B'], 'single'), # second flow dependency already run
79+
FlowTester(['A', 'BReqA'], ['A', 'A', 'B'], 'all'), # second flow dependency reapply
80+
FlowTester(['A', 'BReqA'], ['A', 'B'], 'none'), # second flow dependency no reapply
81+
FlowTester(['A', 'A', 'BReqA'], ['A', 'A', 'A', 'B'], 'all'), # second flow dependency reapply
82+
FlowTester(['A', 'A', 'BReqA'], ['A', 'B'], 'none'), # second flow dependency no reapply
83+
FlowTester(['A', 'A', 'BReqA'], ['A', 'A', 'B'], 'single'), # second flow dependency skip requirements
84+
FlowTester(['A', 'BReqA', 'CReqBReqA'], ['A', 'B', 'C'], 'single'), # two flows depending on earlier flows
85+
FlowTester(['A', 'BReqA', 'CReqBReqA'], ['A', 'B', 'C'], 'none'), # two flows depending on earlier flows
86+
FlowTester(['A', 'BReqA', 'CReqBReqA'], ['A', 'A', 'B', 'A', 'B', 'C'], 'all'), # three flows depending on earlier flows
87+
FlowTester(['CReqBReqA', 'BReqA', 'A'], ['A', 'B', 'C', 'B', 'A'], 'single'), # three flows depending on earlier flows
88+
FlowTester(['CReqBReqA', 'BReqA', 'A'], ['A', 'B', 'C'], 'none'), # three flows depending on earlier flows
89+
FlowTester(['CReqBReqA', 'BReqA', 'A'], ['A', 'B', 'C', 'A', 'B', 'A'], 'all'), # three flows depending on earlier flows
90+
FlowTester(['A', 'CReqBReqA', 'BReqA', 'A'], ['A', 'B', 'C', 'B', 'A'], 'single'), # three flows depending on earlier flows
91+
FlowTester(['A', 'CReqBReqA', 'BReqA', 'A'], ['A', 'B', 'C'], 'none'), # three flows depending on earlier flows
92+
FlowTester(['A', 'CReqBReqA', 'BReqA', 'A'], ['A', 'A', 'B', 'C', 'A', 'B', 'A'], 'all'), # three flows depending on earlier flows
93+
]
94+
95+
@pytest.mark.parametrize('tester', flow_tests)
96+
def test_flows(tester):
97+
success = tester.run()
98+
i = tester.index
99+
expected = tester.expected_pass_order
100+
observed = tester.observed_pass_order
101+
assert success, f'Tester {i} fails: expected ({expected}), observed ({observed})'

test/pytest/test_graph.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ def base_model(output_dir='hls4mlprj_graph_base_model', iotype = 'io_parallel'):
1717
layers = [{'class_name' : 'Input', 'name' : 'layer0_input', 'input_shape' : [1]},
1818
{'class_name' : 'Dense', 'name' : 'layer0', 'n_in' : 1, 'n_out' : 1},
1919
{'class_name' : 'Dense', 'name' : 'layer1', 'n_in' : 1, 'n_out' : 1}]
20-
config = {'HLSConfig':{'Model':{'Precision':'ap_fixed<32,16>','ReuseFactor' : 1}}}
20+
config = {'HLSConfig':{'Model':{'Precision':'ap_fixed<32,16>','ReuseFactor' : 1}, 'Flows': []}}
2121
config['OutputDir'] = output_dir
2222
config['ProjectName'] = 'myprj'
2323
config['IOType'] = iotype

0 commit comments

Comments
 (0)