Merge pull request google#117 from grumpyhome/unvendor-pythonparser

Unvendorize pythonparser (Fixes google#108)

Author: alanjds

grumpy-tools-src/grumpy_tools/compiler/block.py

@@ -24,9 +24,9 @@
 
 from grumpy_tools.compiler import expr
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor.pythonparser import algorithm
-from grumpy_tools.vendor.pythonparser import ast
-from grumpy_tools.vendor.pythonparser import source
+from pythonparser import algorithm
+from pythonparser import ast
+from pythonparser import source
 
 
 _non_word_re = re.compile('[^A-Za-z0-9_]')

grumpy-tools-src/grumpy_tools/compiler/block_test.py

@@ -24,7 +24,7 @@
 from grumpy_tools.compiler import block
 from grumpy_tools.compiler import imputil
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor import pythonparser
+import pythonparser
 
 
 class PackageTest(unittest.TestCase):

grumpy-tools-src/grumpy_tools/compiler/expr_visitor.py

@@ -23,8 +23,8 @@
 
 from grumpy_tools.compiler import expr
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor.pythonparser import algorithm
-from grumpy_tools.vendor.pythonparser import ast
+from pythonparser import algorithm
+from pythonparser import ast
 
 try:
   long           # Python 2

grumpy-tools-src/grumpy_tools/compiler/expr_visitor_test.py

@@ -26,7 +26,7 @@
 from grumpy_tools.compiler import imputil
 from grumpy_tools.compiler import shard_test
 from grumpy_tools.compiler import stmt
-from grumpy_tools.vendor import pythonparser
+import pythonparser
 
 # Handles "Set self.maxDiff to None to see it" annoyance
 unittest.TestCase.maxDiff = None

grumpy-tools-src/grumpy_tools/compiler/imputil.py

@@ -33,10 +33,10 @@
 except ImportError:
     from backports.functools_lru_cache import lru_cache
 
-from grumpy_tools.compiler import util
-from grumpy_tools.vendor import pythonparser
-from grumpy_tools.vendor.pythonparser import algorithm
-from grumpy_tools.vendor.pythonparser import ast
+from grumpy_tools.compiler import util, parser
+import pythonparser
+from pythonparser import algorithm
+from pythonparser import ast
 
 try:
   xrange          # Python 2
@@ -247,6 +247,7 @@ def visit_ImportFrom(self, node):
 
 
 def collect_imports(modname, script, gopath, package_dir=''):
+  parser.patch_pythonparser()
   with open(script) as py_file:
     py_contents = py_file.read()
   mod = pythonparser.parse(py_contents)

grumpy-tools-src/grumpy_tools/compiler/imputil_test.py

@@ -27,7 +27,10 @@
 
 from grumpy_tools.compiler import imputil
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor import pythonparser
+from grumpy_tools.compiler.parser import patch_pythonparser
+import pythonparser
+
+patch_pythonparser()
 
 
 class ImportVisitorTest(unittest.TestCase):

grumpy-tools-src/grumpy_tools/compiler/parser.py

@@ -0,0 +1,79 @@
+# coding: utf-8
+"""
+Grumpy uses `pythonparser` as its AST parser. This module contains an augmented
+(extended) parser from it, letting us to accept special Grumpy-only syntax, like
+the `import '__go__/...'` syntax for importing Go code.
+"""
+import logging
+
+import pythonparser.parser
+from pythonparser.parser import Parser, Seq, Loc, Opt, Tok, List, Alt, Rule, action
+from pythonparser import ast
+
+logger = logging.getLogger(__name__)
+
+PYTHNOPARSER_PATCHED = False
+
+
+def patch_pythonparser():
+    global PYTHNOPARSER_PATCHED
+    if PYTHNOPARSER_PATCHED:
+        return False
+
+    logger.info('Monkeypatching pythonparser.parser.Parser with Grumpy extensions')
+    pythonparser.parser.Parser = GrumpyParser
+    PYTHNOPARSER_PATCHED = True
+    return True
+
+
+class GrumpyParser(Parser):
+    # From: https://github.com/google/grumpy/commit/9d80504e8d42c4a03ece9ed983b0ca160d170969#diff-c46e216e8423951b5f41dde139575b68R1038
+    @action(Rule("atom_5"))
+    def import_from_7(self, string):
+        return (None, 0), (string.loc, string.s)
+
+    # From: https://github.com/google/grumpy/commit/9d80504e8d42c4a03ece9ed983b0ca160d170969#diff-c46e216e8423951b5f41dde139575b68R1046
+    @action(Seq(Loc("from"), Alt(Parser.import_from_3, Parser.import_from_4, import_from_7),
+                Loc("import"), Alt(Parser.import_from_5,
+                                   Seq(Loc("("), Rule("import_as_names"), Loc(")")),
+                                   Parser.import_from_6)))
+    def import_from(self, from_loc, module_name, import_loc, names):
+        """
+        (2.6, 2.7)
+        import_from: ('from' ('.'* dotted_name | '.'+)
+                    'import' ('*' | '(' import_as_names ')' | import_as_names))
+        (3.0-)
+        # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+        import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+                    'import' ('*' | '(' import_as_names ')' | import_as_names))
+        """
+        (dots_loc, dots_count), dotted_name_opt = module_name
+        module_loc = module = None
+        if dotted_name_opt:
+            module_loc, module = dotted_name_opt
+        lparen_loc, names, rparen_loc = names
+        loc = from_loc.join(names[-1].loc)
+        if rparen_loc:
+            loc = loc.join(rparen_loc)
+
+        if module == "__future__":
+            self.add_flags([x.name for x in names])
+
+        return ast.ImportFrom(names=names, module=module, level=dots_count,
+                              keyword_loc=from_loc, dots_loc=dots_loc, module_loc=module_loc,
+                              import_loc=import_loc, lparen_loc=lparen_loc, rparen_loc=rparen_loc,
+                              loc=loc)
+
+    @action(Seq(Rule("atom_5"), Opt(Seq(Loc("as"), Tok("ident")))))
+    def str_as_name(self, string, as_name_opt):
+        asname_name = asname_loc = as_loc = None
+        loc = string.loc
+        if as_name_opt:
+            as_loc, asname = as_name_opt
+            asname_name = asname.value
+            asname_loc = asname.loc
+            loc = loc.join(asname.loc)
+        return ast.alias(name=string.s, asname=asname_name,
+                         loc=loc, name_loc=string.loc, as_loc=as_loc, asname_loc=asname_loc)
+
+    dotted_as_names = List(Alt(Rule("dotted_as_name"), Rule("str_as_name")), ",", trailing=False)

grumpy-tools-src/grumpy_tools/compiler/stmt.py

@@ -27,8 +27,8 @@
 from grumpy_tools.compiler import expr_visitor
 from grumpy_tools.compiler import imputil
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor.pythonparser import algorithm
-from grumpy_tools.vendor.pythonparser import ast
+from pythonparser import algorithm
+from pythonparser import ast
 
 
 _NATIVE_TYPE_PREFIX = 'type_'

grumpy-tools-src/grumpy_tools/compiler/stmt_test.py

@@ -28,8 +28,8 @@
 from grumpy_tools.compiler import shard_test
 from grumpy_tools.compiler import stmt
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor import pythonparser
-from grumpy_tools.vendor.pythonparser import ast
+import pythonparser
+from pythonparser import ast
 
 import pytest
 

grumpy-tools-src/grumpy_tools/grumpc.py

@@ -33,14 +33,16 @@
 from .compiler import imputil
 from .compiler import stmt
 from .compiler import util
-from .vendor import pythonparser
+from .compiler.parser import patch_pythonparser
+import pythonparser
 from .pep_support.pep3147pycache import make_transpiled_module_folders, should_refresh, set_checksum, fixed_keyword
 from . import pydeps
 
 logger = logging.getLogger(__name__)
 
 
 def _parse_and_visit(stream, script, modname):
+  patch_pythonparser()
   gopath = os.environ['GOPATH']
 
   stream.seek(0)