From 36eb5b8c4bef9ab2eabafcf3adabc7772094cef2 Mon Sep 17 00:00:00 2001 From: curtisbucher Date: Sat, 2 May 2020 14:46:19 -0700 Subject: [PATCH 1/4] Fix literal_eval and add tests --- Lib/ast.py | 2 ++ Lib/test/test_ast.py | 6 ++++++ .../next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst | 2 ++ 3 files changed, 10 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst diff --git a/Lib/ast.py b/Lib/ast.py index 401af5647a240c..d74c65f303ed28 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -88,6 +88,8 @@ def _convert(node): node.func.id == 'set' and node.args == node.keywords == []): return set() elif isinstance(node, Dict): + if len(node.keys) != len(node.values): + raise ValueError(f'malformed node or string: {node!r}') return dict(zip(map(_convert, node.keys), map(_convert, node.values))) elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 9063b3d2d7b744..dfb3d0c1e8616e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -965,6 +965,12 @@ def test_literal_eval_complex(self): self.assertRaises(ValueError, ast.literal_eval, '3+(0+6j)') self.assertRaises(ValueError, ast.literal_eval, '-(3+6j)') + def test_literal_eval_malformed(self): + malformed = ast.Dict(keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)]) + self.assertRaises(ValueError, ast.literal_eval, malformed) + malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) + self.assertRaises(ValueError, ast.literal_eval, malformed) + def test_bad_integer(self): # issue13436: Bad error message with invalid numeric values body = [ast.ImportFrom(module='time', diff --git a/Misc/NEWS.d/next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst b/Misc/NEWS.d/next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst new file mode 100644 index 00000000000000..715420e979d480 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst @@ -0,0 +1,2 @@ +Fixed case where malformed :class:`ast.Dict` nodes could have keys or values +thrown away by :func:`ast.literal_eval`. Patch by Curtis Bucher. From 2c724600b2cd442f32cdc6d77e38aadd86375e03 Mon Sep 17 00:00:00 2001 From: curtisbucher Date: Sun, 3 May 2020 14:18:49 -0700 Subject: [PATCH 2/4] Update ast.py Abstracted calling ValueError in ast.py --- Lib/ast.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index d74c65f303ed28..0829f72daa76e5 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -62,11 +62,13 @@ def literal_eval(node_or_string): node_or_string = parse(node_or_string, mode='eval') if isinstance(node_or_string, Expression): node_or_string = node_or_string.body + def _malformed(node): + raise ValueError(f'malformed node or string: {node!r}') def _convert_num(node): if isinstance(node, Constant): if type(node.value) in (int, float, complex): return node.value - raise ValueError('malformed node or string: ' + repr(node)) + _malformed(node) def _convert_signed_num(node): if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): operand = _convert_num(node.operand) @@ -89,7 +91,7 @@ def _convert(node): return set() elif isinstance(node, Dict): if len(node.keys) != len(node.values): - raise ValueError(f'malformed node or string: {node!r}') + _malformed(node) return dict(zip(map(_convert, node.keys), map(_convert, node.values))) elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): From 3e68c2ed8dc871c356a176b1737fa40b07a0dcb2 Mon Sep 17 00:00:00 2001 From: Curtis Bucher Date: Tue, 5 May 2020 11:53:55 -0700 Subject: [PATCH 3/4] Update Misc/NEWS.d/next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst Co-authored-by: Pablo Galindo --- .../next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst b/Misc/NEWS.d/next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst index 715420e979d480..81f9e937a2bff4 100644 --- a/Misc/NEWS.d/next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst +++ b/Misc/NEWS.d/next/Library/2020-05-02-14-24-48.bpo-40355.xTujaB.rst @@ -1,2 +1,2 @@ -Fixed case where malformed :class:`ast.Dict` nodes could have keys or values -thrown away by :func:`ast.literal_eval`. Patch by Curtis Bucher. +Improve error reporting in :func:`ast.literal_eval` in the presence of malformed :class:`ast.Dict` +nodes instead of silently ignoring any non-conforming elements. Patch by Curtis Bucher. From 2427ed2742c3e9d0c2da00c824c8b036042b135c Mon Sep 17 00:00:00 2001 From: curtisbucher Date: Tue, 5 May 2020 12:00:02 -0700 Subject: [PATCH 4/4] Update ast.py and test_ast.py Address requested changes. --- Lib/ast.py | 11 +++++------ Lib/test/test_ast.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index 0829f72daa76e5..2ba9ea8847cbe6 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -62,13 +62,12 @@ def literal_eval(node_or_string): node_or_string = parse(node_or_string, mode='eval') if isinstance(node_or_string, Expression): node_or_string = node_or_string.body - def _malformed(node): + def _raise_malformed_node(node): raise ValueError(f'malformed node or string: {node!r}') def _convert_num(node): - if isinstance(node, Constant): - if type(node.value) in (int, float, complex): - return node.value - _malformed(node) + if not isinstance(node, Constant) or type(node.value) not in (int, float, complex): + _raise_malformed_node(node) + return node.value def _convert_signed_num(node): if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): operand = _convert_num(node.operand) @@ -91,7 +90,7 @@ def _convert(node): return set() elif isinstance(node, Dict): if len(node.keys) != len(node.values): - _malformed(node) + _raise_malformed_node(node) return dict(zip(map(_convert, node.keys), map(_convert, node.values))) elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index dfb3d0c1e8616e..a8a13fdcd7426e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -965,7 +965,7 @@ def test_literal_eval_complex(self): self.assertRaises(ValueError, ast.literal_eval, '3+(0+6j)') self.assertRaises(ValueError, ast.literal_eval, '-(3+6j)') - def test_literal_eval_malformed(self): + def test_literal_eval_malformed_dict_nodes(self): malformed = ast.Dict(keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)]) self.assertRaises(ValueError, ast.literal_eval, malformed) malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)])