Typing information for pyosmium

.travis.yml

@@ -43,29 +43,23 @@ addons:
         packages:
             - libboost1.58-dev
             - python-dev
-            - python-nose
-            - python-mock
             - python3
             - python3-dev
-            - python3-nose
             - python3-setuptools
 
 install:
     - git clone --quiet --depth 1 https://github.com/osmcode/libosmium.git contrib/libosmium
     - git clone --quiet --depth 1 https://github.com/mapbox/protozero.git contrib/protozero
     - git clone --quiet --depth 1 https://github.com/pybind/pybind11.git contrib/pybind11
-    - if [ "$TRAVIS_OS_NAME" = 'osx' ]; then
-          pip${USE_PYTHON_VERSION} install -q nose mock;
-      fi
+    - if [[ $TRAVIS_OS_NAME == 'osx' ]]; then pip${USE_PYTHON_VERSION}  install -U virtualenv ; fi
+    - virtualenv -p python${USE_PYTHON_VERSION} build_env
+    - virtualenv -p python${USE_PYTHON_VERSION} test_env
+    - build_env/bin/pip install -q -r build-requirements.txt
+    - test_env/bin/pip install -q -r test-requirements.txt
 
 script:
-    - if [ "$TRAVIS_OS_NAME" = 'osx' ]; then
-          PYTHON=python${USE_PYTHON_VERSION};
-      else
-          PYTHON=/usr/bin/python${USE_PYTHON_VERSION};
-      fi
-    - $PYTHON --version
-    - $PYTHON setup.py build
+    - build_env/bin/python --version
+    - build_env/bin/python setup.py build
     - cd test
-    - $PYTHON run_tests.py
+    - ../test_env/bin/python run_tests.py
 

build-requirements.txt

@@ -0,0 +1 @@
+mypy>=0.670; python_version > '3.0'

lib/io.cc

@@ -7,6 +7,7 @@ namespace py = pybind11;
 
 PYBIND11_MODULE(io, m)
 {
+    py::module::import("osmium.osm"); // needed to get proper type for osmium::io::Header::box
     py::class_<osmium::io::Header>(m, "Header",
         "File header with global information about the file.")
         .def(py::init<>())

lib/osmium.cc

@@ -25,6 +25,8 @@ PYBIND11_MODULE(_osmium, m) {
         }
     });
 
+    py::module::import("osmium.io"); // needed to get proper type for osmium::io::Reader
+
     m.def("apply", [](osmium::io::Reader &rd, BaseHandler &h)
                    { osmium::apply(rd, h); },
           py::arg("reader"), py::arg("handler"),

setup.py

@@ -2,12 +2,17 @@
 import re
 import sys
 import platform
+import shutil
 import subprocess
+import tempfile
 
 from setuptools import setup, Extension
 from setuptools.command.build_ext import build_ext
 from setuptools.command.sdist import sdist as orig_sdist
+from setuptools.command.build_py import build_py
 from distutils.version import LooseVersion
+from sys import executable, version_info as python_version
+
 
 BASEDIR = os.path.split(os.path.abspath(__file__))[0]
 
@@ -66,6 +71,8 @@ def run(self):
         for ext in self.extensions:
             self.build_extension(ext)
 
+        self.generate_pyi()
+
     def build_extension(self, ext):
         extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
         cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
@@ -110,11 +117,62 @@ def build_extension(self, ext):
         subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env)
         subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp)
 
+    def generate_pyi(self):
+        # set python executable
+        if python_version >= (3, 4, 0):
+            python_name = executable
+        else:
+            if platform == 'win32':
+                python_name = 'py -3'
+            else:
+                python_name = 'python3'
+
+        if python_version[0] < 3:
+            return # no mypy for python2
+
+        # set target directory
+        dst = self.build_lib
+        # test that everything is OK
+        env = os.environ.copy()
+        env['PYTHONPATH'] = dst
+
+        # test if there is no errors in osmium, stubgen doesn't report error on import error
+        import_test = subprocess.Popen([python_name, '-c', 'import osmium'], env=env, stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE)
+        if import_test.wait():
+            error = ""
+            if import_test.stdout:
+                error += "\n".join(x.decode('utf-8') for x in import_test.stdout.readlines())
+            if import_test.stderr:
+                error += "\n".join(x.decode('utf-8') for x in import_test.stderr.readlines())
+            raise Exception("Failure importing osmium: \n" + error)
+
+        # generate pyi files
+        with tempfile.TemporaryDirectory() as tmpdir:
+            p = subprocess.Popen([python_name, '-m', 'mypy.stubgen', '-p', 'osmium', '-o', tmpdir], env=env,
+                                 stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+            retcode = p.wait()
+            error = ""
+            if p.stdout:
+                error += "\n".join(x.decode('utf-8') for x in p.stdout.readlines())
+            if p.stderr:
+                error += "\n".join(x.decode('utf-8') for x in p.stderr.readlines())
+            if retcode:
+                raise Exception("Failure calling stubgen: \n" + error)
+            for pyi in (os.path.join("osmium", "osm", "_osm.pyi"),
+                        os.path.join("osmium", "replication", "_replication.pyi"),
+                        os.path.join("osmium", '_osmium.pyi'),
+                        os.path.join("osmium", 'geom.pyi'),
+                        os.path.join("osmium", 'index.pyi'),
+                        os.path.join("osmium", 'io.pyi')):
+                shutil.copyfile(os.path.join(tmpdir, pyi), os.path.join(dst, pyi))
+
 versions = get_versions()
 
 with open('README.rst', 'r') as descfile:
     long_description = descfile.read()
 
+
 setup(
     name='osmium',
     version=versions['pyosmium_release'],
@@ -146,6 +204,7 @@ def build_extension(self, ext):
     ext_modules=[CMakeExtension('cmake_example')],
     packages = ['osmium', 'osmium/osm', 'osmium/replication'],
     package_dir = {'' : 'src'},
+    package_data={'osmium': ['py.typed']},
     cmdclass=dict(build_ext=CMakeBuild, sdist=Pyosmium_sdist),
     zip_safe=False,
 )

src/osmium/osm/__init__.py

@@ -1,27 +1,38 @@
-from ._osm import *
 import osmium.osm.mutable
+from osmium.osm._osm import *
+
+MYPY = False
+if MYPY:
+    import typing
+
 
 def create_mutable_node(node, **args):
+    #  type: (Node, typing.Any) -> osmium.osm.mutable.Node
     """ Create a mutable node replacing the properties given in the
         named parameters. Note that this function only creates a shallow
         copy which is still bound to the scope of the original object.
     """
     return osmium.osm.mutable.Node(base=node, **args)
 
+
 def create_mutable_way(way, **args):
+    #  type: (Way, typing.Any) -> osmium.osm.mutable.Way
     """ Create a mutable way replacing the properties given in the
         named parameters. Note that this function only creates a shallow
         copy which is still bound to the scope of the original object.
     """
     return osmium.osm.mutable.Way(base=way, **args)
 
+
 def create_mutable_relation(rel, **args):
+    #  type: (Relation, typing.Any) -> osmium.osm.mutable.Relation
     """ Create a mutable relation replacing the properties given in the
         named parameters. Note that this function only creates a shallow
         copy which is still bound to the scope of the original object.
     """
     return osmium.osm.mutable.Relation(base=rel, **args)
 
+
 Node.replace = create_mutable_node
 Way.replace = create_mutable_way
 Relation.replace = create_mutable_relation

src/osmium/osm/mutable.py

@@ -1,3 +1,26 @@
+MYPY = False
+if MYPY:
+    import typing
+    from typing import Optional
+    from ._osm import TagList, Location
+    from ..replication.utils import Timestamp
+    from . import _osm as osm
+
+    OSMObjectIdType = typing.NewType('OSMObjectIdType', int)
+    VersionType = typing.NewType('VersionType', int)
+    UnsginedOSMObjectIdType = typing.NewType('UnsginedOSMObjectIdType', OSMObjectIdType)
+    UidType = typing.NewType('UidType', int)
+    ChangesetId = typing.NewType('ChangesetId', int)
+    LocationType = typing.Union[Location, typing.Tuple[float, float]]
+
+    NodeListType = typing.Union[osm.WayNodeList, typing.List[osm.NodeRef], typing.List[int]]
+    RelationMembersType = typing.Union[
+        osm.RelationMemberList,
+        typing.List[osm.RelationMember],
+        typing.List[typing.Tuple[str, int, str]]
+    ]
+
+
 class OSMObject(object):
     """Mutable version of ``osmium.osm.OSMObject``. It exposes the following
        attributes ``id``, ``version``, ``visible``, ``changeset``, ``timestamp``,
@@ -11,14 +34,15 @@ class OSMObject(object):
 
     def __init__(self, base=None, id=None, version=None, visible=None, changeset=None,
             timestamp=None, uid=None, tags=None):
+        # type: (Optional[typing.Any], Optional[OSMObjectIdType], Optional[VersionType], Optional[bool], Optional[ChangesetId], Optional[Timestamp], Optional[UidType], Optional[typing.Union[TagList, typing.Dict[str, str]]]) -> None
         if base is None:
-            self.id = id
-            self.version = version
-            self.visible = visible
-            self.changeset = changeset
-            self.timestamp = timestamp
-            self.uid = uid
-            self.tags = tags
+            self.id = id  # type: Optional[OSMObjectIdType]
+            self.version = version  # type: Optional[VersionType]
+            self.visible = visible  # type: Optional[bool]
+            self.changeset = changeset  # type: Optional[ChangesetId]
+            self.timestamp = timestamp  # type: Optional[Timestamp]
+            self.uid = uid  # type: Optional[UidType]
+            self.tags = tags  # type: Optional[typing.Union[TagList, typing.Dict[str, str]]]
         else:
             self.id = base.id if id is None else id
             self.version = base.version if version is None else version
@@ -36,12 +60,15 @@ class Node(OSMObject):
     """
 
     def __init__(self, base=None, location=None, **attrs):
+        # type: (Optional[osm.Node], Optional[LocationType], typing.Any) -> None
         OSMObject.__init__(self, base=base, **attrs)
         if base is None:
-            self.location = location
+            self.location = location  # type: Optional[LocationType]
         else:
             self.location = location if location is not None else base.location
 
+    def new(self, some):
+        return "new"
 
 class Way(OSMObject):
     """The mutable version of ``osmium.osm.Way``. It inherits all attributes
@@ -51,12 +78,14 @@ class Way(OSMObject):
     """
 
     def __init__(self, base=None, nodes=None, **attrs):
+        # type: (Optional[osm.Way], Optional[NodeListType], typing.Any) -> None
         OSMObject.__init__(self, base=base, **attrs)
         if base is None:
-            self.nodes = nodes
+            self.nodes = nodes  # type: Optional[NodeListType]
         else:
             self.nodes = nodes if nodes is not None else base.nodes
 
+
 class Relation(OSMObject):
     """The mutable version of ``osmium.osm.Relation``. It inherits all attributes
        from osmium.osm.mutable.OSMObject and adds a `members` attribute. This
@@ -66,9 +95,10 @@ class Relation(OSMObject):
     """
 
     def __init__(self, base=None, members=None, **attrs):
+        # type: (Optional[osm.Relation], Optional[RelationMembersType], typing.Any) -> None
         OSMObject.__init__(self, base=base, **attrs)
         if base is None:
-            self.members = members
+            self.members = members  # type: Optional[RelationMembersType]
         else:
             self.members = members if members is not None else base.members