diff --git a/.travis.yml b/.travis.yml index 22c8e910..068e3ec9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,29 +43,23 @@ addons: packages: - libboost1.58-dev - python-dev - - python-nose - - python-mock - python3 - python3-dev - - python3-nose - python3-setuptools install: - git clone --quiet --depth 1 https://github.com/osmcode/libosmium.git contrib/libosmium - git clone --quiet --depth 1 https://github.com/mapbox/protozero.git contrib/protozero - git clone --quiet --depth 1 https://github.com/pybind/pybind11.git contrib/pybind11 - - if [ "$TRAVIS_OS_NAME" = 'osx' ]; then - pip${USE_PYTHON_VERSION} install -q nose mock; - fi + - if [[ $TRAVIS_OS_NAME == 'osx' ]]; then pip${USE_PYTHON_VERSION} install -U virtualenv ; fi + - virtualenv -p python${USE_PYTHON_VERSION} build_env + - virtualenv -p python${USE_PYTHON_VERSION} test_env + - build_env/bin/pip install -q -r build-requirements.txt + - test_env/bin/pip install -q -r test-requirements.txt script: - - if [ "$TRAVIS_OS_NAME" = 'osx' ]; then - PYTHON=python${USE_PYTHON_VERSION}; - else - PYTHON=/usr/bin/python${USE_PYTHON_VERSION}; - fi - - $PYTHON --version - - $PYTHON setup.py build + - build_env/bin/python --version + - build_env/bin/python setup.py build - cd test - - $PYTHON run_tests.py + - ../test_env/bin/python run_tests.py diff --git a/build-requirements.txt b/build-requirements.txt new file mode 100644 index 00000000..a3bc62a8 --- /dev/null +++ b/build-requirements.txt @@ -0,0 +1 @@ +mypy>=0.670; python_version > '3.0' diff --git a/lib/io.cc b/lib/io.cc index a9c0c439..c1926da2 100644 --- a/lib/io.cc +++ b/lib/io.cc @@ -7,6 +7,7 @@ namespace py = pybind11; PYBIND11_MODULE(io, m) { + py::module::import("osmium.osm"); // needed to get proper type for osmium::io::Header::box py::class_(m, "Header", "File header with global information about the file.") .def(py::init<>()) diff --git a/lib/osmium.cc b/lib/osmium.cc index 7f1a29ff..4e0399a0 100644 --- a/lib/osmium.cc +++ b/lib/osmium.cc @@ -25,6 +25,8 @@ PYBIND11_MODULE(_osmium, m) { } }); + py::module::import("osmium.io"); // needed to get proper type for osmium::io::Reader + m.def("apply", [](osmium::io::Reader &rd, BaseHandler &h) { osmium::apply(rd, h); }, py::arg("reader"), py::arg("handler"), diff --git a/setup.py b/setup.py index 52042460..b2a525f8 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,17 @@ import re import sys import platform +import shutil import subprocess +import tempfile from setuptools import setup, Extension from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist as orig_sdist +from setuptools.command.build_py import build_py from distutils.version import LooseVersion +from sys import executable, version_info as python_version + BASEDIR = os.path.split(os.path.abspath(__file__))[0] @@ -66,6 +71,8 @@ def run(self): for ext in self.extensions: self.build_extension(ext) + self.generate_pyi() + def build_extension(self, ext): extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, @@ -110,11 +117,62 @@ def build_extension(self, ext): subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp) + def generate_pyi(self): + # set python executable + if python_version >= (3, 4, 0): + python_name = executable + else: + if platform == 'win32': + python_name = 'py -3' + else: + python_name = 'python3' + + if python_version[0] < 3: + return # no mypy for python2 + + # set target directory + dst = self.build_lib + # test that everything is OK + env = os.environ.copy() + env['PYTHONPATH'] = dst + + # test if there is no errors in osmium, stubgen doesn't report error on import error + import_test = subprocess.Popen([python_name, '-c', 'import osmium'], env=env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if import_test.wait(): + error = "" + if import_test.stdout: + error += "\n".join(x.decode('utf-8') for x in import_test.stdout.readlines()) + if import_test.stderr: + error += "\n".join(x.decode('utf-8') for x in import_test.stderr.readlines()) + raise Exception("Failure importing osmium: \n" + error) + + # generate pyi files + with tempfile.TemporaryDirectory() as tmpdir: + p = subprocess.Popen([python_name, '-m', 'mypy.stubgen', '-p', 'osmium', '-o', tmpdir], env=env, + stderr=subprocess.PIPE, stdout=subprocess.PIPE) + retcode = p.wait() + error = "" + if p.stdout: + error += "\n".join(x.decode('utf-8') for x in p.stdout.readlines()) + if p.stderr: + error += "\n".join(x.decode('utf-8') for x in p.stderr.readlines()) + if retcode: + raise Exception("Failure calling stubgen: \n" + error) + for pyi in (os.path.join("osmium", "osm", "_osm.pyi"), + os.path.join("osmium", "replication", "_replication.pyi"), + os.path.join("osmium", '_osmium.pyi'), + os.path.join("osmium", 'geom.pyi'), + os.path.join("osmium", 'index.pyi'), + os.path.join("osmium", 'io.pyi')): + shutil.copyfile(os.path.join(tmpdir, pyi), os.path.join(dst, pyi)) + versions = get_versions() with open('README.rst', 'r') as descfile: long_description = descfile.read() + setup( name='osmium', version=versions['pyosmium_release'], @@ -146,6 +204,7 @@ def build_extension(self, ext): ext_modules=[CMakeExtension('cmake_example')], packages = ['osmium', 'osmium/osm', 'osmium/replication'], package_dir = {'' : 'src'}, + package_data={'osmium': ['py.typed']}, cmdclass=dict(build_ext=CMakeBuild, sdist=Pyosmium_sdist), zip_safe=False, ) diff --git a/src/osmium/osm/__init__.py b/src/osmium/osm/__init__.py index c9882278..fff66020 100644 --- a/src/osmium/osm/__init__.py +++ b/src/osmium/osm/__init__.py @@ -1,27 +1,38 @@ -from ._osm import * import osmium.osm.mutable +from osmium.osm._osm import * + +MYPY = False +if MYPY: + import typing + def create_mutable_node(node, **args): + # type: (Node, typing.Any) -> osmium.osm.mutable.Node """ Create a mutable node replacing the properties given in the named parameters. Note that this function only creates a shallow copy which is still bound to the scope of the original object. """ return osmium.osm.mutable.Node(base=node, **args) + def create_mutable_way(way, **args): + # type: (Way, typing.Any) -> osmium.osm.mutable.Way """ Create a mutable way replacing the properties given in the named parameters. Note that this function only creates a shallow copy which is still bound to the scope of the original object. """ return osmium.osm.mutable.Way(base=way, **args) + def create_mutable_relation(rel, **args): + # type: (Relation, typing.Any) -> osmium.osm.mutable.Relation """ Create a mutable relation replacing the properties given in the named parameters. Note that this function only creates a shallow copy which is still bound to the scope of the original object. """ return osmium.osm.mutable.Relation(base=rel, **args) + Node.replace = create_mutable_node Way.replace = create_mutable_way Relation.replace = create_mutable_relation diff --git a/src/osmium/osm/mutable.py b/src/osmium/osm/mutable.py index b700087e..99f87755 100644 --- a/src/osmium/osm/mutable.py +++ b/src/osmium/osm/mutable.py @@ -1,3 +1,26 @@ +MYPY = False +if MYPY: + import typing + from typing import Optional + from ._osm import TagList, Location + from ..replication.utils import Timestamp + from . import _osm as osm + + OSMObjectIdType = typing.NewType('OSMObjectIdType', int) + VersionType = typing.NewType('VersionType', int) + UnsginedOSMObjectIdType = typing.NewType('UnsginedOSMObjectIdType', OSMObjectIdType) + UidType = typing.NewType('UidType', int) + ChangesetId = typing.NewType('ChangesetId', int) + LocationType = typing.Union[Location, typing.Tuple[float, float]] + + NodeListType = typing.Union[osm.WayNodeList, typing.List[osm.NodeRef], typing.List[int]] + RelationMembersType = typing.Union[ + osm.RelationMemberList, + typing.List[osm.RelationMember], + typing.List[typing.Tuple[str, int, str]] + ] + + class OSMObject(object): """Mutable version of ``osmium.osm.OSMObject``. It exposes the following attributes ``id``, ``version``, ``visible``, ``changeset``, ``timestamp``, @@ -11,14 +34,15 @@ class OSMObject(object): def __init__(self, base=None, id=None, version=None, visible=None, changeset=None, timestamp=None, uid=None, tags=None): + # type: (Optional[typing.Any], Optional[OSMObjectIdType], Optional[VersionType], Optional[bool], Optional[ChangesetId], Optional[Timestamp], Optional[UidType], Optional[typing.Union[TagList, typing.Dict[str, str]]]) -> None if base is None: - self.id = id - self.version = version - self.visible = visible - self.changeset = changeset - self.timestamp = timestamp - self.uid = uid - self.tags = tags + self.id = id # type: Optional[OSMObjectIdType] + self.version = version # type: Optional[VersionType] + self.visible = visible # type: Optional[bool] + self.changeset = changeset # type: Optional[ChangesetId] + self.timestamp = timestamp # type: Optional[Timestamp] + self.uid = uid # type: Optional[UidType] + self.tags = tags # type: Optional[typing.Union[TagList, typing.Dict[str, str]]] else: self.id = base.id if id is None else id self.version = base.version if version is None else version @@ -36,12 +60,15 @@ class Node(OSMObject): """ def __init__(self, base=None, location=None, **attrs): + # type: (Optional[osm.Node], Optional[LocationType], typing.Any) -> None OSMObject.__init__(self, base=base, **attrs) if base is None: - self.location = location + self.location = location # type: Optional[LocationType] else: self.location = location if location is not None else base.location + def new(self, some): + return "new" class Way(OSMObject): """The mutable version of ``osmium.osm.Way``. It inherits all attributes @@ -51,12 +78,14 @@ class Way(OSMObject): """ def __init__(self, base=None, nodes=None, **attrs): + # type: (Optional[osm.Way], Optional[NodeListType], typing.Any) -> None OSMObject.__init__(self, base=base, **attrs) if base is None: - self.nodes = nodes + self.nodes = nodes # type: Optional[NodeListType] else: self.nodes = nodes if nodes is not None else base.nodes + class Relation(OSMObject): """The mutable version of ``osmium.osm.Relation``. It inherits all attributes from osmium.osm.mutable.OSMObject and adds a `members` attribute. This @@ -66,9 +95,10 @@ class Relation(OSMObject): """ def __init__(self, base=None, members=None, **attrs): + # type: (Optional[osm.Relation], Optional[RelationMembersType], typing.Any) -> None OSMObject.__init__(self, base=base, **attrs) if base is None: - self.members = members + self.members = members # type: Optional[RelationMembersType] else: self.members = members if members is not None else base.members diff --git a/src/osmium/py.typed b/src/osmium/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/src/osmium/replication/server.py b/src/osmium/replication/server.py index 40de54dc..9ca626f5 100644 --- a/src/osmium/replication/server.py +++ b/src/osmium/replication/server.py @@ -18,6 +18,17 @@ import logging +MYPY = False +if MYPY: + import typing + from ..replication.utils import Sequence, Timestamp + from http.client import HTTPResponse + from .. import BaseHandler + + OsmosisState = typing.NamedTuple('OsmosisState', [('sequence', Sequence), ('timestamp', Timestamp)]) + DownloadResult = typing.NamedTuple('DownloadResult', + [('id', Sequence), ('reader', MergeInputReader), ('newest', Sequence)]) + log = logging.getLogger('pyosmium') log.addHandler(logging.NullHandler()) @@ -31,10 +42,12 @@ class ReplicationServer(object): """ def __init__(self, url, diff_type='osc.gz'): - self.baseurl = url - self.diff_type = diff_type + # type: (str, str) -> None + self.baseurl = url # type: str + self.diff_type = diff_type # type: str def open_url(self, url): + # type: (str) -> HTTPResponse """ Download a resource from the given URL and return a byte sequence of the content. @@ -54,6 +67,7 @@ def my_open_url(self, url): return urlrequest.urlopen(url) def collect_diffs(self, start_id, max_size=1024): + # type: (Sequence, int) -> typing.Optional[DownloadResult] """ Create a MergeInputReader and download diffs starting with sequence id `start_id` into it. `max_size` restricts the number of diffs that are downloaded. The download @@ -84,7 +98,7 @@ def collect_diffs(self, start_id, max_size=1024): try: diffdata = self.get_diff_block(current_id) except: - diffdata = '' + diffdata = b'' if len(diffdata) == 0: if start_id == current_id: return None @@ -98,6 +112,7 @@ def collect_diffs(self, start_id, max_size=1024): return DownloadResult(current_id - 1, rd, newest.sequence) def apply_diffs(self, handler, start_id, max_size=1024, idx="", simplify=True): + # type: (BaseHandler, Sequence, int, str, bool) -> typing.Optional[Sequence] """ Download diffs starting with sequence id `start_id`, merge them together and then apply them to handler `handler`. `max_size` restricts the number of diffs that are downloaded. The download @@ -133,6 +148,7 @@ def apply_diffs(self, handler, start_id, max_size=1024, idx="", simplify=True): def apply_diffs_to_file(self, infile, outfile, start_id, max_size=1024, set_replication_header=True, extra_headers={}): + # type: (str, str, Sequence, int, bool, typing.Dict[str, str]) -> typing.Optional[typing.Tuple[Sequence, Sequence]] """ Download diffs starting with sequence id `start_id`, merge them with the data from the OSM file named `infile` and write the result into a file with the name `outfile`. The output file must not yet @@ -168,6 +184,7 @@ def apply_diffs_to_file(self, infile, outfile, start_id, max_size=1024, h.set("osmosis_replication_base_url", self.baseurl) h.set("osmosis_replication_sequence_number", str(diffs.id)) info = self.get_state_info(diffs.id) + # TODO: what if info is None? h.set("osmosis_replication_timestamp", info.timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")) for k,v in extra_headers.items(): h.set(k, v) @@ -183,8 +200,8 @@ def apply_diffs_to_file(self, infile, outfile, start_id, max_size=1024, return (diffs.id, diffs.newest) - def timestamp_to_sequence(self, timestamp, balanced_search=False): + # type: (Timestamp, bool) -> typing.Optional[Sequence] """ Get the sequence number of the replication file that contains the given timestamp. The search algorithm is optimised for replication servers that publish updates in regular intervals. For servers @@ -265,8 +282,8 @@ def timestamp_to_sequence(self, timestamp, balanced_search=False): if lower.sequence + 1 >= upper.sequence: return lower.sequence - def get_state_info(self, seq=None): + # type: (typing.Optional[Sequence]) -> typing.Optional[OsmosisState] """ Downloads and returns the state information for the given sequence. If the download is successful, a namedtuple with `sequence` and `timestamp` is returned, otherwise the function @@ -302,6 +319,7 @@ def get_state_info(self, seq=None): return OsmosisState(sequence=seq, timestamp=ts) def get_diff_block(self, seq): + # type: (Sequence) -> bytes """ Downloads the diff with the given sequence number and returns it as a byte sequence. Throws a :code:`urllib.error.HTTPError` (or :code:`urllib2.HTTPError` in python2) @@ -311,6 +329,7 @@ def get_diff_block(self, seq): def get_state_url(self, seq): + # type: (typing.Optional[Sequence]) -> str """ Returns the URL of the state.txt files for a given sequence id. If seq is `None` the URL for the latest state info is returned, @@ -325,6 +344,7 @@ def get_state_url(self, seq): def get_diff_url(self, seq): + # type: (Sequence) -> str """ Returns the URL to the diff file for the given sequence id. """ return '%s/%03i/%03i/%03i.%s' % (self.baseurl, diff --git a/src/osmium/replication/utils.py b/src/osmium/replication/utils.py index 48cdc9ce..17a1c468 100644 --- a/src/osmium/replication/utils.py +++ b/src/osmium/replication/utils.py @@ -7,12 +7,24 @@ from osmium.osm import NOTHING from sys import version_info as python_version -log = logging.getLogger('pyosmium') +MYPY = False +if MYPY: + import typing + import datetime + + Sequence = typing.NewType('Sequence', int) + Timestamp = typing.NewType('Timestamp', datetime.datetime) + ReplicationHeader = typing.NamedTuple('ReplicationHeader', + [('url', str), ('sequence', Sequence), ('timestamp', Timestamp)]) + +log = logging.getLogger('pyosmium') # type: logging.Logger ReplicationHeader = namedtuple('ReplicationHeader', ['url', 'sequence', 'timestamp']) + def get_replication_header(fname): + # type: (str) -> ReplicationHeader """ Scans the given file for an Osmosis replication header. It returns a namedtuple with `url`, `sequence` and `timestamp`. Each or all fields may be None, if the piece of information is not avilable. If any of @@ -25,7 +37,7 @@ def get_replication_header(fname): r = oreader(fname, NOTHING) h = r.header() - ts = h.get("osmosis_replication_timestamp") + ts = h.get("osmosis_replication_timestamp") # type: Timestamp url = h.get("osmosis_replication_base_url") if url or ts: @@ -34,7 +46,7 @@ def get_replication_header(fname): if url: log.debug("Replication URL: %s" % url) # the sequence ID is only considered valid, if an URL is given - seq = h.get("osmosis_replication_sequence_number") + seq = h.get("osmosis_replication_sequence_number") # type: Sequence if seq: log.debug("Replication sequence: %s" % seq) try: @@ -54,7 +66,7 @@ def get_replication_header(fname): if ts: log.debug("Replication timestamp: %s" % ts) try: - ts = dt.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ") + ts = dt.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ") # type: Timestamp if python_version >= (3,0): ts = ts.replace(tzinfo=dt.timezone.utc) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..a6786964 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,2 @@ +nose +mock