Fixed system package build scripts and docker files - resolves #25.

Pan · Pan · commit a1af6f4a6ebd · 2018-05-30T10:33:38.000+01:00
Re-enabled system package releases.
Updated documentation.
Added ubuntu 18.04 package build.
Removed redhat 6 and debian 7 builds.
diff --git a/.travis.yml b/.travis.yml
@@ -32,9 +32,8 @@ script:
   - cd doc; make html; cd ..
 jobs:
   include:
-    - stage: OSX wheel build
+    - stage: build packages
       os: osx
-      if: tag IS present
       before_install:
         - brew update
       install:
@@ -60,10 +59,10 @@ jobs:
           fi
       language: generic
       python: skip
-    - stage: OSX wheel build
+
+    - stage: build packages
       os: osx
       osx_image: xcode8
-      if: tag IS present
       before_install:
         - brew update
       install:
@@ -90,10 +89,10 @@ jobs:
           fi
       language: generic
       python: skip
-    - stage: OSX wheel build
+
+    - stage: build packages
       os: osx
       osx_image: xcode6.4
-      if: tag IS present
       before_install:
         - brew update
       install:
@@ -123,29 +122,63 @@ jobs:
           fi
       language: generic
       python: skip
-    # - stage: build_packages
-    #   if: tag IS present
-    #   os: linux
-    #   python: 3.6
-    #   install: skip
-    #   script: skip
-    #   before_deploy:
-    #     - docker --version
-    #     - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
-    #     - ./ci/docker/build-packages.sh
-    #   deploy:
-    #     - provider: releases
-    #       skip_cleanup: true
-    #       api_key:
-    #         secure: i1Dr0k393wXBsCKST5ckeTPdZT+hjkikHCGgMSEkJxw+Q8m/Sgq9bbBalkJZDZDc8t0a/nE4ioEXoYO+PT9wRSpjLbjDVwRoWde5PVHaR3JviiwxULEynFLbvdJ1S2O/zRM37YMVgIIXN/2SWSXvQcQznrrvjtBoV+nZfYHX2WZQruq1nQXcPvTJPeZtCBsWaA3TwbVobnOJdb3TFxnHybN3N4gCTOkurek0V7OSMpjd1qgSzNMDIhjXKf/ZB9bcuusXo2QSnzUDJo3S6QE4FBxKohVom7z4AQ8+iRVPkkrWezBo089vzPzfZN/C5+8xQAUfZ6bVNdS6DfI80+55s6Xj7BFEKQW9Kh3Em0GpipHxdxiBf176xktY9EROkDkwDHlsbE7JRRQUmfmJe+0yCo+gg9uAsz5XHNrQgU5BxKhVzggMA/VU+clJ51WfbYGJAvVs/NjHQb/A9CzYKdBamUvy3YY2dxLbdohR7TZMM0JWYmZcmSiaOiZxrCT3ThlQ/J9o9r6TFWvkVjvt+ozABsx0OvtNrdkp7VvFbSQGvmTzGnPM2O5xqzWrM73Z7g8Ahw1g08FDN0JAO8n/Y0tb/xpVAFBmkQTJpQk7f4kQAHWlZKEYN2wEnX+hptjXfDjMYGX9Tc5vdDQJ3oTxnCt+y7Vl9IplT0a5GTTE0l8Pyc4=
-    #       file_glob: true
-    #       file: '*.{deb,rpm}'
-    #       on:
-    #         repo: ParallelSSH/ssh2-python
-    #         tags: true
-    - stage: build wheels
+
+    - stage: build packages
+      os: osx
+      osx_image: xcode9.3
+      before_install:
+        - brew update
+      install:
+        - brew upgrade openssl
+        - sudo ci/install-ssh2.sh
+        - sudo -H pip2 install -U delocate twine wheel pip setuptools
+        - cp /usr/local/lib/libssh2* .
+        - ls -lh
+        - pip2 wheel .
+      script:
+        - delocate-listdeps --all *.whl
+        - delocate-wheel -v *.whl
+        - delocate-listdeps --all *.whl
+        - ls -l *.whl
+        - rm -f *.dylib
+        - pip2 install --user -v *.whl
+        - pwd; mkdir temp; cd temp; pwd
+        - python -c "from ssh2.session import Session; Session()"
+        - cd ..; pwd
+      after_success:
+        - if [[ ! -z "$TRAVIS_TAG" ]]; then
+              twine upload -u $PYPI_U -p $PYPI_P *.whl;
+          fi
+      language: generic
+      python: skip
+
+    - stage: build packages
+      env:
+        - SYSTEM_PACKAGES=1
+      os: linux
+      language: generic
+      python: skip
+      install: skip
+      script:
+        - docker --version
+        - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+        - ./ci/docker/build-packages.sh
+      deploy:
+        - provider: releases
+          skip_cleanup: true
+          api_key:
+            secure: i1Dr0k393wXBsCKST5ckeTPdZT+hjkikHCGgMSEkJxw+Q8m/Sgq9bbBalkJZDZDc8t0a/nE4ioEXoYO+PT9wRSpjLbjDVwRoWde5PVHaR3JviiwxULEynFLbvdJ1S2O/zRM37YMVgIIXN/2SWSXvQcQznrrvjtBoV+nZfYHX2WZQruq1nQXcPvTJPeZtCBsWaA3TwbVobnOJdb3TFxnHybN3N4gCTOkurek0V7OSMpjd1qgSzNMDIhjXKf/ZB9bcuusXo2QSnzUDJo3S6QE4FBxKohVom7z4AQ8+iRVPkkrWezBo089vzPzfZN/C5+8xQAUfZ6bVNdS6DfI80+55s6Xj7BFEKQW9Kh3Em0GpipHxdxiBf176xktY9EROkDkwDHlsbE7JRRQUmfmJe+0yCo+gg9uAsz5XHNrQgU5BxKhVzggMA/VU+clJ51WfbYGJAvVs/NjHQb/A9CzYKdBamUvy3YY2dxLbdohR7TZMM0JWYmZcmSiaOiZxrCT3ThlQ/J9o9r6TFWvkVjvt+ozABsx0OvtNrdkp7VvFbSQGvmTzGnPM2O5xqzWrM73Z7g8Ahw1g08FDN0JAO8n/Y0tb/xpVAFBmkQTJpQk7f4kQAHWlZKEYN2wEnX+hptjXfDjMYGX9Tc5vdDQJ3oTxnCt+y7Vl9IplT0a5GTTE0l8Pyc4=
+          file_glob: true
+          file: '*.{deb,rpm}'
+          on:
+            repo: ParallelSSH/ssh2-python
+            tags: true
+
+    - stage: build packages
       os: linux
       python: 3.6
+      env:
+        - WHEELS=1
       install:
         - pip install twine
       script:
diff --git a/Changelog.rst b/Changelog.rst
@@ -14,7 +14,9 @@ Changes
 Fixes
 ------
 
-* Removed duplicate definitions in ``c_ssh2.pxd``.
+* Removed duplicate libssh2 definitions.
+* Re-enabled system package releases.
+* System package builds would not work correctly - #25.
 
 
 0.13.0
@@ -24,7 +26,7 @@ Changes
 ---------
 
 * Upgrade embedded ``libssh2`` in binary wheels to latest version plus enhancements.
-* Adds support for ECDSA host keys.
+* Adds support for ECDSA host and client keys.
 * Adds support for SHA-256 host key fingerprints.
 * Added SSH agent forwarding implementation.
 * Windows wheels switched to OpenSSL back end.
diff --git a/ci/appveyor/fix_version.py b/ci/appveyor/fix_version.py
@@ -8,8 +8,8 @@ def get_describe_tag():
     return subprocess.check_output(['git', 'describe', '--tags']).strip().decode('utf-8')
 
 def make_version_file(basedir):
-    # import ipdb; ipdb.set_trace()
-    rev = os.environ['APPVEYOR_REPO_COMMIT']
+    rev = os.environ.get('APPVEYOR_REPO_COMMIT',
+                         subprocess.check_output(['git', 'rev-list', '--max-count=1', 'HEAD']).strip().decode('utf-8'))
     basedir = os.path.abspath(basedir)
     git_desc = get_describe_tag()
     version_json = {'date': datetime.now().isoformat(),
diff --git a/ci/docker/build-packages.sh b/ci/docker/build-packages.sh
@@ -12,9 +12,14 @@ for x in `ls -1d ci/docker/{fedora,centos}*`; do
     docker pull $docker_tag || echo
     docker build --cache-from $docker_tag $x -t $name
     docker tag $name $docker_tag
-    docker push $docker_tag
+    # docker push $docker_tag
     sudo rm -rf build dist
-    docker run -v "$(pwd):/src/" "$name" --rpm-dist $dist -s python -t rpm -d libssh2 -d python setup.py
+    # Fix version used by versioneer to current git tag so the generated .c files
+    # do not cause a version change.
+    python ci/appveyor/fix_version.py .
+    # C files need re-generating
+    sudo rm -f ssh2/*.c
+    docker run -v "$(pwd):/src/" "$name" fpm --rpm-dist $dist -s python -t rpm -d libssh2 -d python setup.py
 done
 
 for x in `ls -1d ci/docker/{debian,ubuntu}*`; do
@@ -23,9 +28,14 @@ for x in `ls -1d ci/docker/{debian,ubuntu}*`; do
     docker pull $docker_tag || echo
     docker build --cache-from $docker_tag $x -t $name
     docker tag $name $docker_tag
-    docker push $docker_tag
+    # docker push $docker_tag
     sudo rm -rf build dist
-    docker run -v "$(pwd):/src/" "$name" --iteration $name -s python -t deb -d libssh2-1 -d python setup.py
+    # Fix version used by versioneer to current git tag so the generated .c files
+    # do not cause a version change.
+    python ci/appveyor/fix_version.py .
+    # C files need re-generating
+    sudo rm -f ssh2/*.c
+    docker run -v "$(pwd):/src/" "$name" fpm --iteration $name -s python -t deb -d libssh2-1 -d python setup.py
 done
 
 sudo chown -R ${USER} *
@@ -41,3 +51,5 @@ for x in *.deb; do
     echo "Package: $x"
     dpkg-deb -c $x
 done
+
+echo "Modified files should be reset now -- run 'git checkout -- .' to do so for all files in the repository."
diff --git a/ci/docker/centos6/Dockerfile b/ci/docker/centos6/Dockerfile
diff --git a/ci/docker/centos7/Dockerfile b/ci/docker/centos7/Dockerfile
@@ -1,4 +1,7 @@
 FROM cdrx/fpm-centos:7
 
 RUN yum -y install libssh2-devel python-devel python-setuptools git
-RUN curl -sLO https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py && pip install -U setuptools wheel
+RUN curl -sLO https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py && pip install -U setuptools wheel && pip install cython
+
+ENV EMBEDDED_LIB 0
+ENV HAVE_AGENT_FWD 0
diff --git a/ci/docker/debian8/Dockerfile b/ci/docker/debian8/Dockerfile
@@ -4,3 +4,6 @@ RUN apt-get -y update
 RUN apt-get -y install python-setuptools python-dev libssh2-1-dev python-pip git
 RUN pip install -U setuptools
 RUN pip install -U pip wheel
+
+ENV EMBEDDED_LIB 1
+ENV HAVE_AGENT_FWD 0
diff --git a/ci/docker/fedora22/Dockerfile b/ci/docker/fedora22/Dockerfile
@@ -1,4 +1,7 @@
 FROM cdrx/fpm-fedora:22
 
 RUN yum -y install libssh2-devel python-devel python-setuptools git
-RUN curl -sLO https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py && pip install -U setuptools wheel
+RUN curl -sLO https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py && pip install -U setuptools wheel && pip install cython
+
+ENV EMBEDDED_LIB 0
+ENV HAVE_AGENT_FWD 0
diff --git a/ci/docker/fedora23/Dockerfile b/ci/docker/fedora23/Dockerfile
@@ -1,4 +1,7 @@
 FROM cdrx/fpm-fedora:23
 
 RUN yum -y install libssh2-devel python-devel python-setuptools git
-RUN curl -sLO https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py && pip install -U setuptools wheel
+RUN curl -sLO https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py && pip install -U setuptools wheel && pip install cython
+
+ENV EMBEDDED_LIB 0
+ENV HAVE_AGENT_FWD 0
diff --git a/ci/docker/fedora24/Dockerfile b/ci/docker/fedora24/Dockerfile
@@ -1,4 +1,7 @@
 FROM cdrx/fpm-fedora:24
 
 RUN yum -y install libssh2-devel python-devel python-setuptools git
-RUN curl -sLO https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py && pip install -U setuptools wheel
+RUN curl -sLO https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm -f get-pip.py && pip install -U setuptools wheel && pip install cython
+
+ENV EMBEDDED_LIB 1
+ENV HAVE_AGENT_FWD 0
diff --git a/ci/docker/ubuntu14.04/Dockerfile b/ci/docker/ubuntu14.04/Dockerfile
@@ -4,3 +4,7 @@ RUN apt-get -y update
 RUN apt-get -y install python-setuptools python-dev libssh2-1-dev python-pip git
 RUN pip install -U setuptools
 RUN pip install -U pip wheel
+RUN pip install cython
+
+ENV EMBEDDED_LIB 0
+ENV HAVE_AGENT_FWD 0
diff --git a/ci/docker/ubuntu16.04/Dockerfile b/ci/docker/ubuntu16.04/Dockerfile
@@ -4,3 +4,6 @@ RUN apt-get -y update
 RUN apt-get -y install python-setuptools python-dev libssh2-1-dev python-pip git
 RUN pip install -U setuptools
 RUN pip install -U pip wheel
+
+ENV EMBEDDED_LIB 0
+ENV HAVE_AGENT_FWD 0
diff --git a/ci/docker/ubuntu18.04/Dockerfile b/ci/docker/ubuntu18.04/Dockerfile
@@ -1,6 +1,9 @@
-FROM cdrx/fpm-debian:7
+FROM cdrx/fpm-ubuntu:18.04
 
 RUN apt-get -y update
 RUN apt-get -y install python-setuptools python-dev libssh2-1-dev python-pip git
 RUN pip install -U setuptools
 RUN pip install -U pip wheel
+
+ENV EMBEDDED_LIB 1
+ENV HAVE_AGENT_FWD 0
diff --git a/doc/installation.rst b/doc/installation.rst
@@ -1,10 +1,12 @@
 Installation
 *************
 
+The recommended installation method is ``pip``.
+
 Pip Binary Packages
 ====================
 
-Binary wheel packages are provided for Linux, OSX and Windows, all Python versions, with ``libssh2`` and its required libraries included.
+Binary wheel packages are provided for Linux, OSX and Windows, all Python versions, with ``libssh2`` and its dependencies included.
 
 Wheel packages have **no dependencies**.
 
@@ -18,14 +20,14 @@ Wheel packages have **no dependencies**.
 
 .. note::
 
-   Latest version of OpenSSL is included in Linux and OSX binary wheel packages. On Windows, the native WinCNG back-end is used instead.
+   Latest available version of OpenSSL at the time the package is built is included in binary wheel packages.
 
    To control which version of OpenSSL is used for the installation either use system packages which use system libraries, the conda package, or install from source.
 
 System Binary Packages
 =======================
 
-System packages can be built for Centos/RedHat 6/7, Ubuntu 14.04/16.04, Debian 7/8 and Fedora 22/23/24 by running `ci/docker/build-packages.sh <https://github.com/ParallelSSH/ssh2-python/blob/master/ci/docker/build-packages.sh>`_ script in the repository's directory, based on Docker.
+System packages can be built for Centos/RedHat 7, Ubuntu 14.04/16.04/18.04, Debian 8 and Fedora 22/23/24 by running `ci/docker/build-packages.sh <https://github.com/ParallelSSH/ssh2-python/blob/master/ci/docker/build-packages.sh>`_ script in the repository's directory, based on Docker.
 
 To use the built packages, install via the system's package manager, for example for Centos/RedHat based systems:
 
@@ -35,9 +37,9 @@ To use the built packages, install via the system's package manager, for example
 
 .. note::
 
-  System packages use the system provided ``libssh2`` which may need to be updated to be compatible with ``ssh2-python``. ``libssh2`` versions ``>= 1.6.0`` are compatible.
+  System packages as built by the above script use system provided ``libssh2`` and do not have all features enabled as most distributions do not have a new enough version. In addition, there are known issues with older versions of ``libssh2`` like what is provided by distributions.
 
-  To build an ``ssh2-python`` that is compatible with versions lower than ``1.6.0``, run the build with the ``EMBEDDED_LIB=0`` environment variable set. This will disable features that require ``libssh2`` >= ``1.6.0``.
+  For best compatibility, it is recommended to install binary packages with ``pip``.
 
 Conda package
 ===============
@@ -76,16 +78,17 @@ If there are multiple development headers and/or libraries for ``libssh2`` on th
 
    git clone --recurse-submodules git@github.com:ParallelSSH/ssh2-python.git
    sudo ./ci/install-ssh2.sh
+   virtualenv my_env
    source my_env/bin/activate
-   python setup.py build_ext -I /usr/local/include -R /usr/local/lib -L /usr/local/lib
+   python setup.py build_ext -I /usr/local/include -R /usr/local/lib/x86_64-linux-gnu -L /usr/local/lib/x86_64-linux-gnu
    python setup.py install
 
 System library build
 ---------------------
 
-Building against system provided ``libssh2`` is another option which may be preferred. 
+Building against system provided ``libssh2`` is another option which may be preferred.
 
-If the ``libssh2`` version provided by the system is not compatible, run the build with the ``EMBEDDED_LIB=0`` environment variable set. This will disable features that require ``libssh2`` >= ``1.6.0``.
+If the ``libssh2`` version provided by the system is not compatible, run the build with the ``EMBEDDED_LIB=0`` and ``HAVE_AGENT_FWD=0`` environment variables set. This will disable features that require ``libssh2`` >= ``1.6.0`` as well as agent forwarding implementation which is only present in the ``libssh2`` submodule of this repository.
 
 Clone the repository, install dependencies and run install in a new virtualenv from the repository's root directory.
 
diff --git a/setup.py b/setup.py
@@ -32,17 +32,24 @@
 
 # _comp_args = ["-ggdb"]
 _comp_args = ["-O3"] if not ON_WINDOWS else None
-_embedded_lib = bool(os.environ.get('EMBEDDED_LIB', 1))
+_embedded_lib = bool(int(os.environ.get('EMBEDDED_LIB', 1)))
+_have_agent_fwd = bool(int(os.environ.get('HAVE_AGENT_FWD', 1)))
 cython_directives = {'embedsignature': True,
                      'boundscheck': False,
                      'optimize.use_switch': True,
                      'wraparound': False,
 }
 cython_args = {
     'cython_directives': cython_directives,
-    'cython_compile_time_env': {'EMBEDDED_LIB': _embedded_lib}} \
+    'cython_compile_time_env': {
+        'EMBEDDED_LIB': _embedded_lib,
+        'HAVE_AGENT_FWD': _have_agent_fwd,
+    }} \
     if USING_CYTHON else {}
 
+if USING_CYTHON:
+    sys.stdout.write("Cython arguments: %s%s" % (cython_args, os.linesep))
+
 extensions = [
     Extension(sources[i].split('.')[0].replace(os.path.sep, '.'),
               sources=[sources[i]],
diff --git a/ssh2/c_ssh2.pxd b/ssh2/c_ssh2.pxd
@@ -441,5 +441,5 @@ cdef extern from "libssh2.h" nogil:
     int libssh2_trace_sethandler(LIBSSH2_SESSION *session,
                                  void* context,
                                  libssh2_trace_handler_func callback)
-    IF EMBEDDED_LIB:
+    IF HAVE_AGENT_FWD:
         int libssh2_channel_request_auth_agent(LIBSSH2_CHANNEL *channel)
diff --git a/ssh2/channel.pyx b/ssh2/channel.pyx
@@ -466,9 +466,10 @@ cdef class Channel:
             c_ssh2.libssh2_channel_handle_extended_data(
                 self._channel, ignore_mode)
 
-    def request_auth_agent(self):
-        """Request SSH agent authentication forwarding on channel."""
-        cdef int rc
-        with nogil:
-            rc = c_ssh2.libssh2_channel_request_auth_agent(self._channel)
-        return handle_error_codes(rc)
+    IF HAVE_AGENT_FWD:
+        def request_auth_agent(self):
+            """Request SSH agent authentication forwarding on channel."""
+            cdef int rc
+            with nogil:
+                rc = c_ssh2.libssh2_channel_request_auth_agent(self._channel)
+            return handle_error_codes(rc)
