[KBJ] First round of cleanups

.travis.yml

@@ -4,6 +4,11 @@ os:
 
 language: c
 
+stages:
+  - test
+  - name: deploy
+    if: tag IS present
+
 env:
   matrix:
     - CONDA_PY=3.6
@@ -14,53 +19,68 @@ env:
     - TWINE_USERNAME=grepall
     - secure: bTbky3Un19NAl62lix8bMLmBv9IGNhFkRXlZH+B253nYub7jwQwPQKum3ct9ea+XHJT5//uM0B8WAF6eyugpNkPQ7+S7SEH5BJuCt30nv6qvGhSO2AffZKeHEDnfW2kqGrivn87TqeomlSBlO742CD/V0wOIUwkTT9tutd+E7FU=
 
-_deploy_common: &deploy_common
-  if: branch = master AND type = push AND fork = false
+_cibw_common: &cibw_common
+  addons: {}
   install:
-    - python3 -m pip install cibuildwheel twine
+    - python3 -m pip install cibuildwheel>=1.1.0 twine
+  script:
+    - set -e
+    - cibuildwheel --output-dir dist
+    - twine check dist/*
+    - twine upload --skip-existing dist/*
+
+_cibw_linux: &cibw_linux
+  stage: deploy
+  os: linux
+  language: python
+  python: '3.5'
+  services:
+    - docker
+  <<: *cibw_common
 
 matrix:
-#   include:
-#     - stage: deploy
-#       os: linux
-#       language: python
-#       python: '3.5'
-#       services:
-#         - docker
-#       env:
-#         - CIBW_BEFORE_BUILD="yum install -y zlib-devel bzip2-devel xz-devel && pip install -r requirements.txt"
-#         - CIBW_ENVIRONMENT='HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"'
-#         - CIBW_TEST_COMMAND='python -c "import pysam"'
-#       addons:
-#         apt:
-#           packages:
-#             - gcc
-#             - g++
-#             - libcurl4-openssl-dev  # for libcurl support in sdist
-#             - libssl-dev  # for s3 support in sdist
-#       <<: *deploy_common
-#       script:
-#         - set -e
-#         - cibuildwheel --output-dir dist
-#         - python3 -m pip install Cython
-#         - python3 setup.py build_ext --inplace
-#         - python3 setup.py sdist
-#         - twine check dist/*
-# #        - twine upload --skip-existing dist/*
-#     - stage: deploy
-#       os: osx
-#       language: generic
-#       env:
-#         - CIBW_BEFORE_BUILD="pip install -r requirements.txt"
-#         - CIBW_ENVIRONMENT='HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"'
-#         - CIBW_TEST_COMMAND='python -c "import pysam"'
-#       addons: {}
-#       <<: *deploy_common
-#       script:
-#         - set -e
-#         - cibuildwheel --output-dir dist
-#         - twine check dist/*
-# #       - twine upload --skip-existing dist/*
+  include:
+    - stage: deploy
+      os: linux
+      language: python
+      python: '3.5'
+      addons:
+        apt:
+          packages:
+            - gcc
+            - g++
+            - libcurl4-openssl-dev  # for libcurl support in sdist
+            - libssl-dev  # for s3 support in sdist
+      install:
+        - python3 -m pip install Cython twine
+      script:
+        - set -e
+        - python3 setup.py build_ext --inplace
+        - python3 setup.py sdist
+        - twine check dist/*
+        - twine upload --skip-existing dist/*
+    - <<: *cibw_linux
+      env:
+        - CIBW_BUILD="*_x86_64"
+        - CIBW_BEFORE_BUILD="yum install -y zlib-devel bzip2-devel xz-devel && python -m pip install -r requirements.txt"
+        - CIBW_ENVIRONMENT='HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"'
+        - CIBW_REPAIR_WHEEL_COMMAND_LINUX='auditwheel repair -L . -w {dest_dir} {wheel}'
+        - CIBW_TEST_COMMAND='python -c "import pysam"'
+    - <<: *cibw_linux
+      env:
+        - CIBW_BUILD="*_i686"
+        - CIBW_BEFORE_BUILD="yum install -y zlib-devel bzip2-devel xz-devel && python -m pip install -r requirements.txt"
+        - CIBW_ENVIRONMENT='HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"'
+        - CIBW_REPAIR_WHEEL_COMMAND_LINUX='auditwheel repair -L . -w {dest_dir} {wheel}'
+        - CIBW_TEST_COMMAND='python -c "import pysam"'
+    - stage: deploy
+      os: osx
+      language: generic
+      env:
+        - CIBW_BEFORE_BUILD="python -m pip install -r requirements.txt"
+        - CIBW_ENVIRONMENT='HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"'
+        - CIBW_TEST_COMMAND='python -c "import pysam"'
+      <<: *cibw_common
 
 addons:
   apt:

devtools/import.py

@@ -1,27 +1,11 @@
 #################################################################
-# Importing samtools and htslib
+# Importing samtools, bcftools, and htslib
 #
-# For htslib, simply copy the whole release tar-ball
-# into the directory "htslib" and recreate the file version.h
+# For each package PKG:
 #
-# rm -rf htslib
-# mv download/htslib htslib
-# git checkout -- htslib/version.h
-# Edit the file htslib/version.h to set the right version number.
-#
-# For samtools, type:
-# rm -rf samtools
-# python import.py samtools download/samtools
-# git checkout -- samtools/version.h
-#
-# Manually, then:
-# modify config.h to set compatibility flags
-#
-# For bcftools, type:
-# rm -rf bcftools
-# python import.py bcftools download/bedtools
-# git checkout -- bcftools/version.h
-# rm -rf bedtools/test bedtools/plugins
+# rm -rf PKG
+# python import.py PKG path/to/download/PKG-X.Y
+# git checkout -- PKG/version.h
 
 import fnmatch
 import os
@@ -34,23 +18,19 @@
 
 EXCLUDE = {
     "samtools": (
+        "test", "misc",
         "razip.c",
         "bgzip.c",
         "main.c",
         "calDepth.c",
         "bam2bed.c",
-        "wgsim.c",
         "bam_tview.c",
         "bam_tview.h",
         "bam_tview_html.c",
         "bam_tview_curses.c",
-        "md5fa.c",
-        "md5sum-lite.c",
-        "maq2sam.c",
         "bamcheck.c",
         "chk_indel.c",
         "vcf-miniview.c",
-        "hfile_irods.c",  # requires irods library
     ),
     "bcftools": (
         "test", "plugins", "peakfit.c",
@@ -60,7 +40,8 @@
         "polysomy.c"),
     "htslib": (
         'htslib/tabix.c', 'htslib/bgzip.c',
-        'htslib/htsfile.c', 'htslib/hfile_irods.c'),
+        'htslib/htsfile.c',
+        "test"),
 }
 
 
@@ -71,13 +52,21 @@
 
 
 
-def locate(pattern, root=os.curdir):
-    '''Locate all files matching supplied filename pattern in and below
-    supplied root directory.
+def locate(pattern, root=os.curdir, exclude=[], exclude_htslib=False):
+    '''Locate all files matching supplied filename pattern (but not listed
+    in exclude) in and below the supplied root directory. Omit any files under
+    directories listed in exclude or (if exclude_htslib=True) matching /htslib-*/.
     '''
     for path, dirs, files in os.walk(os.path.abspath(root)):
         for filename in fnmatch.filter(files, pattern):
-            yield os.path.join(path, filename)
+            if filename not in exclude:
+                yield os.path.join(path, filename)
+
+        for dirname in exclude:
+            if dirname in dirs: dirs.remove(dirname)
+        if exclude_htslib:
+            for dirname in [d for d in dirs if re.match(r"htslib-", d)]:
+                dirs.remove(dirname)
 
 
 def _update_pysam_files(cf, destdir):
@@ -149,16 +138,15 @@ def _update_pysam_files(cf, destdir):
         raise IOError(
             "source directory `%s` does not exist." % srcdir)
 
-    cfiles = locate("*.c", srcdir)
-    hfiles = locate("*.h", srcdir)
+    cfiles = locate("*.c", srcdir, exclude=exclude, exclude_htslib=True)
+    hfiles = locate("*.h", srcdir, exclude=exclude, exclude_htslib=True)
     mfiles = itertools.chain(locate("README", srcdir), locate("LICENSE", srcdir))
-
-    # remove unwanted files and htslib subdirectory.
-    cfiles = [x for x in cfiles if os.path.basename(x) not in exclude
-              and not re.search("htslib-", x)]
 
-    hfiles = [x for x in hfiles if os.path.basename(x) not in exclude
-              and not re.search("htslib-", x)]
+    if dest == "htslib":
+        # Add build files, including *.ac *.in *.mk *.m4
+        mfiles = itertools.chain(mfiles, locate("Makefile", srcdir),
+                                 locate("configure", srcdir),
+                                 locate("*.[aim][cnk4]", srcdir))
 
     ncopied = 0
 
@@ -203,10 +191,12 @@ def _compareAndCopy(src, srcdir, destdir, exclude):
     sys.stdout.write(
         "installed latest source code from %s: "
         "%i files copied\n" % (srcdir, ncopied))
-    # redirect stderr to pysamerr and replace bam.h with a stub.
-    sys.stdout.write("applying stderr redirection\n")
 
-    _update_pysam_files(cf, destdir)
+    if dest in MAIN:
+        # redirect stderr to pysamerr and replace bam.h with a stub.
+        sys.stdout.write("applying stderr redirection\n")
+
+        _update_pysam_files(cf, destdir)
 
     sys.exit(0)
 

devtools/run_tests_travis.sh

@@ -17,9 +17,9 @@ WORKDIR=`pwd`
 
 #Install miniconda python
 if [ $TRAVIS_OS_NAME == "osx" ]; then
-	wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O Miniconda3.sh
+	wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O Miniconda3.sh
 else
-	wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda3.sh --no-check-certificate  # Default OS versions are old and have SSL / CERT issues
+	wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda3.sh --no-check-certificate  # Default OS versions are old and have SSL / CERT issues
 fi
 
 bash Miniconda3.sh -b
@@ -40,9 +40,10 @@ conda config --add channels conda-forge
 # NB: we force conda-forge:ncurses due to bioconda/bioconda-recipes#13488
 conda install -y "samtools=1.9" "bcftools=1.9" "htslib=1.9" xz curl bzip2 conda-forge:ncurses
 
-# Need to make C compiler and linker use the anaconda includes and libraries:
-export PREFIX=~/miniconda3/
-export CFLAGS="-I${PREFIX}/include -L${PREFIX}/lib"
+# As HTSLIB_MODE is (defaulted to) 'shared', ensure we don't pick up
+# the external headers from the Conda-installed htslib package.
+mv $CONDA_PREFIX/include/htslib $CONDA_PREFIX/include/htslib-disable
+
 export HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"
 
 echo "show samtools, htslib, and bcftools versions"

doc/faq.rst

@@ -40,7 +40,7 @@ parts have not been fully tested.
 A related issue is when different threads read from the same file
 objec - or the same thread uses two iterators over a file. There is
 only a single file-position for each opened file. To prevent this from
-hapeding, use the option ``mulitple_iterator=True`` when calling
+hapeding, use the option ``multiple_iterator=True`` when calling
 a fetch() method. This will return an iterator on a newly opened
 file.
 
@@ -109,7 +109,7 @@ Here, the output is::
 The reason for this behaviour is that every iterator needs to keep
 track of its current position in the file. Within pysam, each opened
 file can only keep track of one file position and hence there can only
-be one iterator per file. Using the option ``mulitple_iterators=True``
+be one iterator per file. Using the option ``multiple_iterators=True``
 will return an iterator within a newly opened file. This iterator will
 not interfere with existing iterators as it has its own file handle
 associated with it.

doc/release.rst

@@ -2,6 +2,26 @@
 Release notes
 =============
 
+Release 0.15.4
+==============
+
+Bugfix release. Principal reason for release is to update cython
+version in order to fix pip install pysam with python 3.8.
+
+* [#879] Fix add_meta function in libcbcf.pyx, so meta-information
+  lines in header added with this function have double-quoting rules
+  in accordance to rules specified in VCF4.2 and VCF4.3 specifications
+* [#863] Force arg to bytes to support non-ASCII encoding
+* [#875] Bump minimum Cython version
+* [#868] Prevent segfault on Python 2.7 AlignedSegment.compare(other=None)
+* [#867] Fix wheel building on TravisCI
+* [#863] Force arg to bytes to support non-ASCII encoding
+* [#799] disambiguate interpretation of bcf_read return code
+* [#841] Fix silent truncation of FASTQ with bad q strings
+* [#846] Prevent segmentation fault on ID, when handling malformed records
+* [#829] Run configure with the correct CC/CFLAGS/LDFLAGS env vars
+
+
 Release 0.15.3
 ==============