Skip to content

Commit 43bb546

Browse files
will-keenimgtec-admin
authored andcommitted
Merge pull request #10 from imaginationtech/rand_len_list
Add sanitization for bad list lengths
2 parents 32ddc6b + 072e6b6 commit 43bb546

File tree

4 files changed

+185
-118
lines changed

4 files changed

+185
-118
lines changed

constrainedrandom/internal/randvar.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ def set_rand_length(self, length: int) -> None:
275275
if self.rand_length is None:
276276
raise RuntimeError("RandVar was not marked as having a random length," \
277277
" but set_rand_length was called.")
278+
assert length >= 0, "length was not greater than or equal to zero"
278279
self.rand_length_val = length
279280

280281
def _get_random(self) -> random.Random:

constrainedrandom/randobj.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,8 @@ def my_fn(factor):
209209
# rand_length and length are mutually-exclusive.
210210
assert not ((length is not None) and (rand_length is not None)), \
211211
"length and rand_length are mutually-exclusive, but both were specified"
212+
if length is not None:
213+
assert length >= 0, "length was not greater than or equal to zero"
212214
if rand_length is not None:
213215
# Indicates the length of the RandVar depends on another random variable.
214216
assert rand_length in self._random_vars, f"random variable length '{name}' is not valid," \

tests/features/errors.py

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class ImpossibleThorough(MultiSum):
2323
solver.
2424
'''
2525

26-
EXPECT_FAILURE = True
26+
EXPECTED_ERROR_RAND = RandomizationError
2727

2828
def get_randobj(self, *args):
2929
randobj = super().get_randobj(*args)
@@ -66,7 +66,7 @@ class ImpossibleComplexVar(testutils.RandObjTestBase):
6666
the variable state space is too large to fail on creation.
6767
'''
6868

69-
EXPECT_FAILURE = True
69+
EXPECTED_ERROR_RAND = RandomizationError
7070

7171
def get_randobj(self, *args):
7272
randobj = RandObj(*args)
@@ -81,7 +81,7 @@ class ImpossibleMultiVar(testutils.RandObjTestBase):
8181
Test an impossible constraint problem with multiple variables.
8282
'''
8383

84-
EXPECT_FAILURE = True
84+
EXPECTED_ERROR_RAND = RandomizationError
8585

8686
def get_randobj(self, *args):
8787
randobj = RandObj(*args)
@@ -93,3 +93,49 @@ def sum_gt_10(x, y):
9393
return x + y > 10
9494
randobj.add_constraint(sum_gt_10, ('a', 'b'))
9595
return randobj
96+
97+
98+
class NegativeLength(testutils.RandObjTestBase):
99+
'''
100+
Test a random list with negative length.
101+
'''
102+
103+
EXPECTED_ERROR_INIT = AssertionError
104+
105+
def get_randobj(self, *args):
106+
randobj = RandObj(*args)
107+
randobj.add_rand_var('bad_list', bits=1, length=-1)
108+
return randobj
109+
110+
111+
class NegativeRandLength(testutils.RandObjTestBase):
112+
'''
113+
Test a random list with negative random length.
114+
'''
115+
116+
EXPECTED_ERROR_RAND = AssertionError
117+
118+
def get_randobj(self, *args):
119+
randobj = RandObj(*args)
120+
randobj.add_rand_var('bad_length', domain=range(-10,-1))
121+
randobj.add_rand_var('bad_list', bits=1, rand_length='bad_length')
122+
return randobj
123+
124+
125+
class EmptyListDependent(testutils.RandObjTestBase):
126+
'''
127+
Test a random length list which is empty
128+
but depended on by another variable.
129+
'''
130+
131+
EXPECTED_ERROR_RAND = RandomizationError
132+
133+
def get_randobj(self, *args):
134+
randobj = RandObj(*args)
135+
randobj.add_rand_var('length', domain=[0])
136+
randobj.add_rand_var('list', bits=4, rand_length='length')
137+
randobj.add_rand_var('list_member', bits=4)
138+
def in_list_c(x, y):
139+
return x in y
140+
randobj.add_constraint(in_list_c, ('list_member', 'list'))
141+
return randobj

tests/testutils.py

Lines changed: 133 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,14 @@ class RandObjTestBase(unittest.TestCase):
2727
Extend this class to create testcases.
2828
'''
2929

30+
# Number of iterations to do per call to `randomize_and_check_result`.
3031
ITERATIONS = 1000
32+
# Global multiplier for test length, set via command line.
3133
TEST_LENGTH_MULTIPLIER = 1
32-
EXPECT_FAILURE = False
34+
# Expected exception when calling `get_randobj`.
35+
EXPECTED_ERROR_INIT = None
36+
# Expected exception when calling `randomize_and_check_result`.
37+
EXPECTED_ERROR_RAND = None
3338

3439
def setUp(self) -> None:
3540
self.iterations = self.ITERATIONS * self.TEST_LENGTH_MULTIPLIER
@@ -111,8 +116,8 @@ def randomize_and_check_result(
111116
Code to randomize a randobj and check its results against expected
112117
results.
113118
'''
114-
if self.EXPECT_FAILURE:
115-
self.assertRaises(RandomizationError, randobj.randomize)
119+
if self.EXPECTED_ERROR_RAND is not None:
120+
self.assertRaises(self.EXPECTED_ERROR_RAND, randobj.randomize)
116121
else:
117122
results = self.randomize_and_time(randobj, self.iterations)
118123
assertListOfDictsEqual(self, expected_results, results, "Non-determinism detected, results were not equal")
@@ -164,132 +169,145 @@ def test_randobj(self) -> None:
164169

165170
# Test with seed 0
166171
r = random.Random(0)
167-
randobj = self.get_randobj(r)
168-
# Take a copy of the randobj for use later
169-
randobj_copy = deepcopy(randobj)
170-
if self.EXPECT_FAILURE:
171-
self.assertRaises(RandomizationError, randobj.randomize)
172+
if self.EXPECTED_ERROR_INIT is not None:
173+
self.assertRaises(self.EXPECTED_ERROR_INIT, self.get_randobj, r)
172174
else:
173-
results = self.randomize_and_time(randobj, self.iterations)
174-
self.check(results)
175-
if do_tmp_checks:
176-
# Check when applying temporary constraints
177-
tmp_results = self.randomize_and_time(randobj, self.iterations, tmp_constraints, tmp_values)
178-
self.tmp_check(tmp_results)
179-
# Check temporary constraints don't break base randomization
180-
post_tmp_results = self.randomize_and_time(randobj, self.iterations)
181-
self.check(post_tmp_results)
182-
# Add temporary constraints permanently, see what happens
183-
if tmp_constraints is not None:
184-
for constr, vars in tmp_constraints:
185-
randobj.add_constraint(constr, vars)
186-
add_results = self.randomize_and_time(randobj, self.iterations, tmp_values=tmp_values)
187-
self.tmp_check(add_results)
175+
randobj = self.get_randobj(r)
176+
# Take a copy of the randobj for use later
177+
randobj_copy = deepcopy(randobj)
178+
if self.EXPECTED_ERROR_RAND is not None:
179+
self.assertRaises(self.EXPECTED_ERROR_RAND, randobj.randomize)
180+
else:
181+
results = self.randomize_and_time(randobj, self.iterations)
182+
self.check(results)
183+
if do_tmp_checks:
184+
# Check when applying temporary constraints
185+
tmp_results = self.randomize_and_time(randobj, self.iterations, tmp_constraints, tmp_values)
186+
self.tmp_check(tmp_results)
187+
# Check temporary constraints don't break base randomization
188+
post_tmp_results = self.randomize_and_time(randobj, self.iterations)
189+
self.check(post_tmp_results)
190+
# Add temporary constraints permanently, see what happens
191+
if tmp_constraints is not None:
192+
for constr, vars in tmp_constraints:
193+
randobj.add_constraint(constr, vars)
194+
add_results = self.randomize_and_time(randobj, self.iterations, tmp_values=tmp_values)
195+
self.tmp_check(add_results)
188196

189197
# Test again with seed 0, ensuring results are the same.
190198
r0 = random.Random(0)
191-
randobj0 = self.get_randobj(r0)
192-
self.randomize_and_check_result(
193-
randobj0,
194-
results,
195-
do_tmp_checks,
196-
tmp_constraints,
197-
tmp_values,
198-
tmp_results,
199-
post_tmp_results,
200-
add_results,
201-
add_tmp_constraints=True,
202-
)
199+
if self.EXPECTED_ERROR_INIT is not None:
200+
self.assertRaises(self.EXPECTED_ERROR_INIT, self.get_randobj, r0)
201+
else:
202+
randobj0 = self.get_randobj(r0)
203+
self.randomize_and_check_result(
204+
randobj0,
205+
results,
206+
do_tmp_checks,
207+
tmp_constraints,
208+
tmp_values,
209+
tmp_results,
210+
post_tmp_results,
211+
add_results,
212+
add_tmp_constraints=True,
213+
)
203214

204215
# Also test the copy we took earlier.
205-
self.randomize_and_check_result(
206-
randobj_copy,
207-
results,
208-
do_tmp_checks,
209-
tmp_constraints,
210-
tmp_values,
211-
tmp_results,
212-
post_tmp_results,
213-
add_results,
214-
add_tmp_constraints=True,
215-
)
216+
if self.EXPECTED_ERROR_INIT is None:
217+
self.randomize_and_check_result(
218+
randobj_copy,
219+
results,
220+
do_tmp_checks,
221+
tmp_constraints,
222+
tmp_values,
223+
tmp_results,
224+
post_tmp_results,
225+
add_results,
226+
add_tmp_constraints=True,
227+
)
216228

217229
# Test with seed 1, ensuring results are different
218230
r1 = random.Random(1)
219-
randobj1 = self.get_randobj(r1)
220-
if self.EXPECT_FAILURE:
221-
self.assertRaises(RandomizationError, randobj1.randomize)
231+
if self.EXPECTED_ERROR_INIT is not None:
232+
self.assertRaises(self.EXPECTED_ERROR_INIT, self.get_randobj, r1)
222233
else:
223-
results1 = self.randomize_and_time(randobj1, self.iterations)
224-
self.check(results1)
225-
self.assertNotEqual(results, results1, "Results were the same for two different seeds, check testcase.")
226-
if do_tmp_checks:
227-
# Check results are also different when applying temporary constraints
228-
tmp_results1 = self.randomize_and_time(randobj1, self.iterations, tmp_constraints, tmp_values)
229-
self.tmp_check(tmp_results1)
230-
self.assertNotEqual(tmp_results, tmp_results1,
231-
"Results were the same for two different seeds, check testcase.")
234+
randobj1 = self.get_randobj(r1)
235+
if self.EXPECTED_ERROR_RAND is not None:
236+
self.assertRaises(self.EXPECTED_ERROR_RAND, randobj1.randomize)
237+
else:
238+
results1 = self.randomize_and_time(randobj1, self.iterations)
239+
self.check(results1)
240+
self.assertNotEqual(results, results1, "Results were the same for two different seeds, check testcase.")
241+
if do_tmp_checks:
242+
# Check results are also different when applying temporary constraints
243+
tmp_results1 = self.randomize_and_time(randobj1, self.iterations, tmp_constraints, tmp_values)
244+
self.tmp_check(tmp_results1)
245+
self.assertNotEqual(tmp_results, tmp_results1,
246+
"Results were the same for two different seeds, check testcase.")
232247

233248
# Test using global seeding, ensuring results are the same
234249
# Don't add temp constraints this time, so that we can test this object again.
235250
random.seed(0)
236-
randobj0_global = self.get_randobj()
237-
self.randomize_and_check_result(
238-
randobj0_global,
239-
results,
240-
do_tmp_checks,
241-
tmp_constraints,
242-
tmp_values,
243-
tmp_results,
244-
post_tmp_results,
245-
add_results,
246-
add_tmp_constraints=False,
247-
)
251+
if self.EXPECTED_ERROR_INIT is not None:
252+
self.assertRaises(self.EXPECTED_ERROR_INIT, self.get_randobj)
253+
else:
254+
randobj0_global = self.get_randobj()
255+
self.randomize_and_check_result(
256+
randobj0_global,
257+
results,
258+
do_tmp_checks,
259+
tmp_constraints,
260+
tmp_values,
261+
tmp_results,
262+
post_tmp_results,
263+
add_results,
264+
add_tmp_constraints=False,
265+
)
248266

249-
# Re-test the the globally-seeded object
250-
# Must re-seed the global random module to ensure repeatability.
251-
random.seed(0)
252-
self.randomize_and_check_result(
253-
randobj0_global,
254-
results,
255-
do_tmp_checks,
256-
tmp_constraints,
257-
tmp_values,
258-
tmp_results,
259-
post_tmp_results,
260-
add_results,
261-
add_tmp_constraints=True,
262-
)
267+
# Re-test the the globally-seeded object
268+
# Must re-seed the global random module to ensure repeatability.
269+
random.seed(0)
270+
self.randomize_and_check_result(
271+
randobj0_global,
272+
results,
273+
do_tmp_checks,
274+
tmp_constraints,
275+
tmp_values,
276+
tmp_results,
277+
post_tmp_results,
278+
add_results,
279+
add_tmp_constraints=True,
280+
)
263281

264-
# TODO: Fix interaction between global random module and deepcopy.
265-
# Details:
266-
# There is an issue around copying an object that relies on the
267-
# global random object - the state of any copied object is tied to
268-
# its original.
269-
# Having spent a lot of time debugging this issue, it is still very
270-
# difficult to understand.
271-
# Each individual copied RandObj instance points to a new random.Random
272-
# instance, which shares state with the global module. It appears then
273-
# in some instances that the object uses the global value in the random
274-
# module, and in others it uses the copied one, meaning the state
275-
# diverges.
276-
# Right now, all I can conclude is it would take a lot of work to
277-
# fully debug it, and it can be worked around by passing objects a
278-
# seeded random.Random if the user desires reproducible objects.
282+
# TODO: Fix interaction between global random module and deepcopy.
283+
# Details:
284+
# There is an issue around copying an object that relies on the
285+
# global random object - the state of any copied object is tied to
286+
# its original.
287+
# Having spent a lot of time debugging this issue, it is still very
288+
# difficult to understand.
289+
# Each individual copied RandObj instance points to a new random.Random
290+
# instance, which shares state with the global module. It appears then
291+
# in some instances that the object uses the global value in the random
292+
# module, and in others it uses the copied one, meaning the state
293+
# diverges.
294+
# Right now, all I can conclude is it would take a lot of work to
295+
# fully debug it, and it can be worked around by passing objects a
296+
# seeded random.Random if the user desires reproducible objects.
279297

280-
# TODO: Make testing of copy more thorough when above issue fixed.
281-
# Take a copy, to show that we can. Its behaviour can't be guaranteed
282-
# to be deterministic w.r.t. randobj0_global due to issues around
283-
# deepcopy interacting with the global random module.
284-
# So, just test it can randomize.
285-
randobj0_global_copy = deepcopy(randobj0_global)
286-
if self.EXPECT_FAILURE:
287-
self.assertRaises(RandomizationError, randobj0_global_copy.randomize)
288-
else:
289-
# Don't check results. Checks may fail due to the interaction
290-
# between deepcopy and global random. E.g. if we check that temp
291-
# constraints are not followed when not supplied, they may
292-
# be due to the interaction between random and deepcopy.
293-
# This just ensures it doesn't crash.
294-
self.randomize_and_time(randobj0_global_copy, self.iterations)
295-
self.randomize_and_time(randobj0_global_copy, self.iterations, tmp_constraints, tmp_values)
298+
# TODO: Make testing of copy more thorough when above issue fixed.
299+
# Take a copy, to show that we can. Its behaviour can't be guaranteed
300+
# to be deterministic w.r.t. randobj0_global due to issues around
301+
# deepcopy interacting with the global random module.
302+
# So, just test it can randomize.
303+
randobj0_global_copy = deepcopy(randobj0_global)
304+
if self.EXPECTED_ERROR_RAND is not None:
305+
self.assertRaises(self.EXPECTED_ERROR_RAND, randobj0_global_copy.randomize)
306+
else:
307+
# Don't check results. Checks may fail due to the interaction
308+
# between deepcopy and global random. E.g. if we check that temp
309+
# constraints are not followed when not supplied, they may
310+
# be due to the interaction between random and deepcopy.
311+
# This just ensures it doesn't crash.
312+
self.randomize_and_time(randobj0_global_copy, self.iterations)
313+
self.randomize_and_time(randobj0_global_copy, self.iterations, tmp_constraints, tmp_values)

0 commit comments

Comments
 (0)