Merge branch 'master' into fix_4792

Author: mroeschke

ci/requirements-2.7.run

@@ -18,6 +18,5 @@ patsy
 pymysql=0.6.3
 html5lib=1.0b2
 beautiful-soup=4.2.1
-statsmodels
 jinja2=2.8
 xarray=0.8.0

ci/requirements-2.7_COMPAT.run

@@ -4,7 +4,6 @@ pytz=2013b
 scipy=0.11.0
 xlwt=0.7.5
 xlrd=0.9.2
-statsmodels=0.4.3
 bottleneck=0.8.0
 numexpr=2.2.2
 pytables=3.0.0

ci/requirements-2.7_LOCALE.run

@@ -13,5 +13,4 @@ html5lib=1.0b2
 lxml=3.2.1
 scipy=0.11.0
 beautiful-soup=4.2.1
-statsmodels=0.4.3
 bigquery=2.0.17

ci/requirements-2.7_SLOW.run

@@ -4,7 +4,6 @@ numpy=1.8.2
 matplotlib=1.3.1
 scipy
 patsy
-statsmodels
 xlwt
 openpyxl
 xlsxwriter

ci/requirements-3.4_SLOW.pip

@@ -0,0 +1,3 @@
+httplib2
+google-api-python-client
+oauth2client

ci/requirements-3.4_SLOW.run

@@ -17,5 +17,4 @@ sqlalchemy
 bottleneck
 pymysql
 psycopg2
-statsmodels
 jinja2=2.8

doc/source/install.rst

@@ -250,9 +250,9 @@ Optional Dependencies
 * `Feather Format <https://github.com/wesm/feather>`__: necessary for feather-based storage, version 0.3.1 or higher.
 * `SQLAlchemy <http://www.sqlalchemy.org>`__: for SQL database support. Version 0.8.1 or higher recommended. Besides SQLAlchemy, you also need a database specific driver. You can find an overview of supported drivers for each SQL dialect in the `SQLAlchemy docs <http://docs.sqlalchemy.org/en/latest/dialects/index.html>`__. Some common drivers are:
 
-    - `psycopg2 <http://initd.org/psycopg/>`__: for PostgreSQL
-    - `pymysql <https://github.com/PyMySQL/PyMySQL>`__: for MySQL.
-    - `SQLite <https://docs.python.org/3.5/library/sqlite3.html>`__: for SQLite, this is included in Python's standard library by default.
+  * `psycopg2 <http://initd.org/psycopg/>`__: for PostgreSQL
+  * `pymysql <https://github.com/PyMySQL/PyMySQL>`__: for MySQL.
+  * `SQLite <https://docs.python.org/3.5/library/sqlite3.html>`__: for SQLite, this is included in Python's standard library by default.
 
 * `matplotlib <http://matplotlib.org/>`__: for plotting
 * For Excel I/O:
@@ -272,11 +272,8 @@ Optional Dependencies
   <http://www.vergenet.net/~conrad/software/xsel/>`__, or `xclip
   <https://github.com/astrand/xclip/>`__: necessary to use
   :func:`~pandas.read_clipboard`. Most package managers on Linux distributions will have ``xclip`` and/or ``xsel`` immediately available for installation.
-* Google's `python-gflags <<https://github.com/google/python-gflags/>`__ ,
-  `oauth2client <https://github.com/google/oauth2client>`__ ,
-  `httplib2 <http://pypi.python.org/pypi/httplib2>`__
-  and `google-api-python-client <http://github.com/google/google-api-python-client>`__
-  : Needed for :mod:`~pandas.io.gbq`
+* For Google BigQuery I/O - see :ref:`here <io.bigquery_deps>`.
+
 * `Backports.lzma <https://pypi.python.org/pypi/backports.lzma/>`__: Only for Python 2, for writing to and/or reading from an xz compressed DataFrame in CSV; Python 3 support is built into the standard library.
 * One of the following combinations of libraries is needed to use the
   top-level :func:`~pandas.read_html` function:

doc/source/io.rst

@@ -39,7 +39,7 @@ object.
     * :ref:`read_json<io.json_reader>`
     * :ref:`read_msgpack<io.msgpack>`
     * :ref:`read_html<io.read_html>`
-    * :ref:`read_gbq<io.bigquery_reader>`
+    * :ref:`read_gbq<io.bigquery>`
     * :ref:`read_stata<io.stata_reader>`
     * :ref:`read_sas<io.sas_reader>`
     * :ref:`read_clipboard<io.clipboard>`
@@ -55,7 +55,7 @@ The corresponding ``writer`` functions are object methods that are accessed like
     * :ref:`to_json<io.json_writer>`
     * :ref:`to_msgpack<io.msgpack>`
     * :ref:`to_html<io.html>`
-    * :ref:`to_gbq<io.bigquery_writer>`
+    * :ref:`to_gbq<io.bigquery>`
     * :ref:`to_stata<io.stata_writer>`
     * :ref:`to_clipboard<io.clipboard>`
     * :ref:`to_pickle<io.pickle>`
@@ -4648,16 +4648,11 @@ DataFrame with a shape and data types derived from the source table.
 Additionally, DataFrames can be inserted into new BigQuery tables or appended
 to existing tables.
 
-You will need to install some additional dependencies:
-
-- Google's `python-gflags <https://github.com/google/python-gflags/>`__
-- `httplib2 <http://pypi.python.org/pypi/httplib2>`__
-- `google-api-python-client <http://github.com/google/google-api-python-client>`__
-
 .. warning::
 
    To use this module, you will need a valid BigQuery account. Refer to the
-   `BigQuery Documentation <https://cloud.google.com/bigquery/what-is-bigquery>`__ for details on the service itself.
+   `BigQuery Documentation <https://cloud.google.com/bigquery/what-is-bigquery>`__
+   for details on the service itself.
 
 The key functions are:
 
@@ -4671,7 +4666,44 @@ The key functions are:
 
 .. currentmodule:: pandas
 
-.. _io.bigquery_reader:
+
+Supported Data Types
+++++++++++++++++++++
+
+Pandas supports all these `BigQuery data types <https://cloud.google.com/bigquery/data-types>`__:
+``STRING``, ``INTEGER`` (64bit), ``FLOAT`` (64 bit), ``BOOLEAN`` and
+``TIMESTAMP`` (microsecond precision). Data types ``BYTES`` and ``RECORD``
+are not supported.
+
+Integer and boolean ``NA`` handling
++++++++++++++++++++++++++++++++++++
+
+.. versionadded:: 0.20
+
+Since all columns in BigQuery queries are nullable, and NumPy lacks of ``NA``
+support for integer and boolean types, this module will store ``INTEGER`` or
+``BOOLEAN`` columns with at least one ``NULL`` value as ``dtype=object``.
+Otherwise those columns will be stored as ``dtype=int64`` or ``dtype=bool``
+respectively.
+
+This is opposite to default pandas behaviour which will promote integer
+type to float in order to store NAs. See the :ref:`gotchas<gotchas.intna>`
+for detailed explaination.
+
+While this trade-off works well for most cases, it breaks down for storing
+values greater than 2**53. Such values in BigQuery can represent identifiers
+and unnoticed precision lost for identifier is what we want to avoid.
+
+.. _io.bigquery_deps:
+
+Dependencies
+++++++++++++
+
+This module requires following additional dependencies:
+
+- `httplib2 <https://github.com/httplib2/httplib2>`__: HTTP client
+- `google-api-python-client <http://github.com/google/google-api-python-client>`__: Google's API client
+- `oauth2client <https://github.com/google/oauth2client>`__: authentication and authorization for Google's API
 
 .. _io.bigquery_authentication:
 
@@ -4686,7 +4718,7 @@ Is possible to authenticate with either user account credentials or service acco
 Authenticating with user account credentials is as simple as following the prompts in a browser window
 which will be automatically opened for you. You will be authenticated to the specified
 ``BigQuery`` account using the product name ``pandas GBQ``. It is only possible on local host.
-The remote authentication using user account credentials is not currently supported in Pandas.
+The remote authentication using user account credentials is not currently supported in pandas.
 Additional information on the authentication mechanism can be found
 `here <https://developers.google.com/identity/protocols/OAuth2#clientside/>`__.
 
@@ -4695,8 +4727,6 @@ is particularly useful when working on remote servers (eg. jupyter iPython noteb
 Additional information on service accounts can be found
 `here <https://developers.google.com/identity/protocols/OAuth2#serviceaccount>`__.
 
-You will need to install an additional dependency: `oauth2client <https://github.com/google/oauth2client>`__.
-
 Authentication via ``application default credentials`` is also possible. This is only valid
 if the parameter ``private_key`` is not provided. This method also requires that
 the credentials can be fetched from the environment the code is running in.
@@ -4716,6 +4746,7 @@ Additional information on
    A private key can be obtained from the Google developers console by clicking
    `here <https://console.developers.google.com/permissions/serviceaccounts>`__. Use JSON key type.
 
+.. _io.bigquery_reader:
 
 Querying
 ''''''''
@@ -4775,7 +4806,6 @@ For more information about query configuration parameters see
 
 .. _io.bigquery_writer:
 
-
 Writing DataFrames
 ''''''''''''''''''
 
@@ -4865,6 +4895,8 @@ For example:
    often as the service seems to be changing and evolving. BiqQuery is best for analyzing large
    sets of data quickly, but it is not a direct replacement for a transactional database.
 
+.. _io.bigquery_create_tables:
+
 Creating BigQuery Tables
 ''''''''''''''''''''''''
 
@@ -4894,6 +4926,7 @@ produce the dictionary representation schema of the specified pandas DataFrame.
    the new table with a different name. Refer to
    `Google BigQuery issue 191 <https://code.google.com/p/google-bigquery/issues/detail?id=191>`__.
 
+
 .. _io.stata:
 
 Stata Format

doc/source/whatsnew/v0.20.0.txt

@@ -96,6 +96,9 @@ support for bz2 compression in the python 2 c-engine improved (:issue:`14874`).
 
 .. _whatsnew_0200.enhancements.uint64_support:
 
+UInt64 Support Improved
+^^^^^^^^^^^^^^^^^^^^^^^
+
 Pandas has significantly improved support for operations involving unsigned,
 or purely non-negative, integers. Previously, handling these integers would
 result in improper rounding or data-type casting, leading to incorrect results.
@@ -146,6 +149,7 @@ Other enhancements
 - ``Series/DataFrame.asfreq()`` have gained a ``fill_value`` parameter, to fill missing values (:issue:`3715`).
 - ``Series/DataFrame.resample.asfreq`` have gained a ``fill_value`` parameter, to fill missing values during resampling (:issue:`3715`).
 - ``pandas.tools.hashing`` has gained a ``hash_tuples`` routine, and ``hash_pandas_object`` has gained the ability to hash a ``MultiIndex`` (:issue:`15224`)
+- ``Series/DataFrame.squeeze()`` have gained the ``axis`` parameter. (:issue:`15339`)
 
 .. _ISO 8601 duration: https://en.wikipedia.org/wiki/ISO_8601#Durations
 
@@ -418,7 +422,8 @@ Other API Changes
 - ``pd.read_csv()`` will now raise a ``ValueError`` for the C engine if the quote character is larger than than one byte (:issue:`11592`)
 - ``inplace`` arguments now require a boolean value, else a ``ValueError`` is thrown (:issue:`14189`)
 - ``pandas.api.types.is_datetime64_ns_dtype`` will now report ``True`` on a tz-aware dtype, similar to ``pandas.api.types.is_datetime64_any_dtype``
- - ``DataFrame.asof()`` will return a null filled ``Series`` instead the scalar ``NaN`` if a match is not found (:issue:`15118`)
+- ``DataFrame.asof()`` will return a null filled ``Series`` instead the scalar ``NaN`` if a match is not found (:issue:`15118`)
+- The :func:`pd.read_gbq` method now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no more casted to ``int64`` which also caused precision lost (:issue: `14064`, :issue:`14305`).
 
 .. _whatsnew_0200.deprecations:
 
@@ -446,7 +451,7 @@ Removal of prior version deprecations/changes
 - The ``pandas.io.ga`` module with a ``google-analytics`` interface is removed (:issue:`11308`).
   Similar functionality can be found in the `Google2Pandas <https://github.com/panalysis/Google2Pandas>`__ package.
 - ``pd.to_datetime`` and ``pd.to_timedelta`` have dropped the ``coerce`` parameter in favor of ``errors`` (:issue:`13602`)
-
+- ``pandas.stats.fama_macbeth``, ``pandas.stats.ols``, ``pandas.stats.plm`` and ``pandas.stats.var``, as well as the top-level ``pandas.fama_macbeth`` and ``pandas.ols`` routines are removed. Similar functionaility can be found in the `statsmodels <shttp://www.statsmodels.org/dev/>`__ package. (:issue:`11898`)
 
 
 
@@ -485,9 +490,11 @@ Bug Fixes
 - Bug in ``pd.read_csv()`` in which missing data was being improperly handled with ``usecols`` (:issue:`6710`)
 - Bug in ``pd.read_csv()`` in which a file containing a row with many columns followed by rows with fewer columns would cause a crash (:issue:`14125`)
 - Bug in ``pd.tools.hashing.hash_pandas_object()`` in which hashing of categoricals depended on the ordering of categories, instead of just their values. (:issue:`15143`)
+- Bug in ``.groupby(..).resample()`` when passed the ``on=`` kwarg. (:issue:`15021`)
 
 - Bug in ``DataFrame.loc`` with indexing a ``MultiIndex`` with a ``Series`` indexer (:issue:`14730`)
 
+
 - Bug in ``pd.read_msgpack()`` in which ``Series`` categoricals were being improperly processed (:issue:`14901`)
 - Bug in ``Series.ffill()`` with mixed dtypes containing tz-aware datetimes. (:issue:`14956`)
 
@@ -566,3 +573,4 @@ Bug Fixes
 
 - Bug in ``DataFrame.boxplot`` where ``fontsize`` was not applied to the tick labels on both axes (:issue:`15108`)
 - Bug in ``Series.replace`` and ``DataFrame.replace`` which failed on empty replacement dicts (:issue:`15289`)
+- Bug in ``.eval()`` which caused multiline evals to fail with local variables not on the first line (:issue:`15342`)

pandas/__init__.py

@@ -15,7 +15,8 @@
         missing_dependencies.append(dependency)
 
 if missing_dependencies:
-    raise ImportError("Missing required dependencies {0}".format(missing_dependencies))
+    raise ImportError(
+        "Missing required dependencies {0}".format(missing_dependencies))
 del hard_dependencies, dependency, missing_dependencies
 
 # numpy compat
@@ -24,7 +25,8 @@
 try:
     from pandas import hashtable, tslib, lib
 except ImportError as e:  # pragma: no cover
-    module = str(e).lstrip('cannot import name ')  # hack but overkill to use re
+    # hack but overkill to use re
+    module = str(e).lstrip('cannot import name ')
     raise ImportError("C extension: {0} not built. If you want to import "
                       "pandas from the source directory, you may need to run "
                       "'python setup.py build_ext --inplace --force' to build "
@@ -61,5 +63,5 @@
 # use the closest tagged version if possible
 from ._version import get_versions
 v = get_versions()
-__version__ = v.get('closest-tag',v['version'])
+__version__ = v.get('closest-tag', v['version'])
 del get_versions, v

pandas/_version.py

@@ -157,7 +157,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # "stabilization", as well as "HEAD" and "master".
         tags = set([r for r in refs if re.search(r'\d', r)])
         if verbose:
-            print("discarding '%s', no digits" % ",".join(refs-tags))
+            print("discarding '%s', no digits" % ",".join(refs - tags))
     if verbose:
         print("likely tags: %s" % ",".join(sorted(tags)))
     for ref in sorted(tags):

pandas/api/tests/test_api.py

@@ -8,8 +8,6 @@
 from pandas.api import types
 from pandas.util import testing as tm
 
-_multiprocess_can_split_ = True
-
 
 class Base(object):
 
@@ -44,7 +42,7 @@ class TestPDApi(Base, tm.TestCase):
                      'json', 'lib', 'index', 'parser']
 
     # these are already deprecated; awaiting removal
-    deprecated_modules = ['ols', 'stats', 'datetools']
+    deprecated_modules = ['stats', 'datetools']
 
     # misc
     misc = ['IndexSlice', 'NaT']
@@ -111,7 +109,7 @@ class TestPDApi(Base, tm.TestCase):
                         'expanding_max', 'expanding_mean', 'expanding_median',
                         'expanding_min', 'expanding_quantile',
                         'expanding_skew', 'expanding_std', 'expanding_sum',
-                        'expanding_var', 'fama_macbeth', 'rolling_apply',
+                        'expanding_var', 'rolling_apply',
                         'rolling_corr', 'rolling_count', 'rolling_cov',
                         'rolling_kurt', 'rolling_max', 'rolling_mean',
                         'rolling_median', 'rolling_min', 'rolling_quantile',
@@ -227,8 +225,3 @@ def test_deprecation_access_obj(self):
         with tm.assert_produces_warning(FutureWarning,
                                         check_stacklevel=False):
             pd.datetools.monthEnd
-
-if __name__ == '__main__':
-    import nose
-    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
-                   exit=False)