Add Logging Handler

Author: Bill Prin

CONTRIBUTING.rst

@@ -160,7 +160,8 @@ Running System Tests
     can be downloaded directly from the developer's console by clicking
     "Generate new JSON key". See private key
     `docs <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`__
-    for more details.
+    for more details. In order for Logging system tests to work, the Service Account
+    will also have to be made a project Owner. This can be changed under "IAM & Admin".
 
 - Examples of these can be found in ``system_tests/local_test_setup.sample``. We
   recommend copying this to ``system_tests/local_test_setup``, editing the

docs/index.rst

@@ -110,6 +110,7 @@
   logging-entries
   logging-metric
   logging-sink
+  logging-handlers
 
 .. toctree::
   :maxdepth: 0

docs/logging-handlers.rst

@@ -0,0 +1,7 @@
+Python Logging Module Handler
+==============================
+
+.. automodule:: gcloud.logging.handlers
+  :members:
+  :show-inheritance:
+

docs/logging-usage.rst

@@ -377,3 +377,45 @@ Delete a sink:
    >>> sink.delete()  # API call
    >>> sink.exists()  # API call
    False
+
+Integration with Python logging module
+---------------------------------------------
+
+
+It's possible to tie the Python :mod:`logging` module directly into Google Cloud Logging. To use it,
+create a :class:`CloudLoggingHandler <gcloud.logging.CloudLoggingHandler>` instance from your
+Logging client.
+
+.. doctest::
+
+    >>> import logging
+    >>> import gcloud.logging # Don't conflict with standard logging
+    >>> from gcloud.logging.handlers import CloudLoggingHandler
+    >>> client = gcloud.logging.Client()
+    >>> handler = CloudLoggingHandler(client)
+    >>> cloud_logger = logging.getLogger('cloudLogger')
+    >>> cloud_logger.setLevel(logging.INFO) # defaults to WARN
+    >>> cloud_logger.addHandler(handler)
+    >>> cloud_logger.error('bad news') # API call
+
+.. note::
+
+    This handler currently only supports a synchronous API call, which means each logging statement
+    that uses this handler will require an API call.
+
+It is also possible to attach the handler to the root Python logger, so that for example a plain
+`logging.warn` call would be sent to Cloud Logging, as well as any other loggers created. However,
+you must avoid infinite recursion from the logging calls the client itself makes. A helper
+method :meth:`setup_logging <gcloud.logging.handlers.setup_logging>` is provided to configure
+this automatically:
+
+.. doctest::
+
+    >>> import logging
+    >>> import gcloud.logging # Don't conflict with standard logging
+    >>> from gcloud.logging.handlers import CloudLoggingHandler, setup_logging
+    >>> client = gcloud.logging.Client()
+    >>> handler = CloudLoggingHandler(client)
+    >>> logging.getLogger().setLevel(logging.INFO) # defaults to WARN
+    >>> setup_logging(handler)
+    >>> logging.error('bad news') # API call

gcloud/logging/handlers.py

@@ -0,0 +1,110 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Python :mod:`logging` handlers for Google Cloud Logging."""
+
+import logging
+
+EXCLUDE_LOGGER_DEFAULTS = (
+    'gcloud',
+    'oauth2client.client'
+)
+
+
+class CloudLoggingHandler(logging.StreamHandler, object):
+    """Python standard logging handler to log messages to the Google Cloud
+    Logging API.
+
+    This handler can be used to route Python standard logging messages to
+    Google Cloud logging.
+
+    Note that this handler currently only supports a synchronous API call,
+    which means each logging statement that uses this handler will require
+    an API call.
+
+    :type client: :class:`gcloud.logging.client`
+    :param client: the authenticated gcloud logging client for this handler
+                   to use
+
+    Example:
+
+    .. doctest::
+
+        import gcloud.logging
+        from gcloud.logging.handlers import CloudLoggingHandler
+
+        client = gcloud.logging.Client()
+        handler = CloudLoggingHandler(client)
+
+        cloud_logger = logging.getLogger('cloudLogger')
+        cloud_logger.setLevel(logging.INFO)
+        cloud_logger.addHandler(handler)
+
+        cloud.logger.error("bad news") # API call
+
+    """
+
+    def __init__(self, client):
+        super(CloudLoggingHandler, self).__init__()
+        self.client = client
+
+    def emit(self, record):
+        """
+        Overrides the default emit behavior of StreamHandler.
+
+        See: https://docs.python.org/2/library/logging.html#handler-objects
+        """
+        message = super(CloudLoggingHandler, self).format(record)
+        logger = self.client.logger(record.name)
+        logger.log_struct({"message": message},
+                          severity=record.levelname)
+
+
+def setup_logging(handler, excluded_loggers=EXCLUDE_LOGGER_DEFAULTS):
+    """Helper function to attach the CloudLoggingAPI handler to the Python
+    root logger, while excluding loggers this library itself uses to avoid
+    infinite recursion
+
+    :type handler: :class:`logging.handler`
+    :param handler: the handler to attach to the global handler
+
+    :type excluded_loggers: tuple
+    :param excluded_loggers: The loggers to not attach the handler to. This
+                             will always include the loggers in the path of
+                             the logging client itself.
+
+    Example:
+
+    .. doctest::
+
+        import logging
+        import gcloud.logging
+        from gcloud.logging.handlers import CloudLoggingAPIHandler
+
+        client = gcloud.logging.Client()
+        handler = CloudLoggingHandler(client)
+        setup_logging(handler)
+        logging.getLogger().setLevel(logging.DEBUG)
+
+        logging.error("bad news") # API call
+
+    """
+    all_excluded_loggers = set(excluded_loggers + EXCLUDE_LOGGER_DEFAULTS)
+    logger = logging.getLogger()
+    logger.addHandler(handler)
+    logger.addHandler(logging.StreamHandler())
+    for logger_name in all_excluded_loggers:
+        logger = logging.getLogger(logger_name)
+        logger.propagate = False
+        logger.addHandler(logging.StreamHandler())

gcloud/logging/test_handlers.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import unittest2
+
+
+class TestCloudLoggingHandler(unittest2.TestCase):
+
+    PROJECT = 'PROJECT'
+
+    def _getTargetClass(self):
+        from gcloud.logging.handlers import CloudLoggingHandler
+        return CloudLoggingHandler
+
+    def _makeOne(self, *args, **kw):
+        return self._getTargetClass()(*args, **kw)
+
+    def test_ctor(self):
+        client = _Client(self.PROJECT)
+        handler = self._makeOne(client)
+        self.assertEqual(handler.client, client)
+
+    def test_emit(self):
+        client = _Client(self.PROJECT)
+        handler = self._makeOne(client)
+        LOGNAME = 'loggername'
+        MESSAGE = 'hello world'
+        record = _Record(LOGNAME, logging.INFO, MESSAGE)
+        handler.emit(record)
+        self.assertEqual(client.logger(LOGNAME).log_struct_called_with,
+                         ({'message': MESSAGE}, logging.INFO))
+
+
+class TestSetupLogging(unittest2.TestCase):
+
+    def _callFUT(self, handler, excludes=None):
+        from gcloud.logging.handlers import setup_logging
+        if excludes:
+            return setup_logging(handler, excluded_loggers=excludes)
+        else:
+            return setup_logging(handler)
+
+    def test_setup_logging(self):
+        handler = _Handler(logging.INFO)
+        self._callFUT(handler)
+
+        root_handlers = logging.getLogger().handlers
+        self.assertIn(handler, root_handlers)
+
+    def test_setup_logging_excludes(self):
+        INCLUDED_LOGGER_NAME = 'includeme'
+        EXCLUDED_LOGGER_NAME = 'excludeme'
+
+        handler = _Handler(logging.INFO)
+        self._callFUT(handler, (EXCLUDED_LOGGER_NAME,))
+
+        included_logger = logging.getLogger(INCLUDED_LOGGER_NAME)
+        self.assertTrue(included_logger.propagate)
+
+        excluded_logger = logging.getLogger(EXCLUDED_LOGGER_NAME)
+        self.assertNotIn(handler, excluded_logger.handlers)
+        self.assertFalse(excluded_logger.propagate)
+
+    def setUp(self):
+        self._handlers_cache = logging.getLogger().handlers[:]
+
+    def tearDown(self):
+        # cleanup handlers
+        logging.getLogger().handlers = self._handlers_cache[:]
+
+
+class _Handler(object):
+
+    def __init__(self, level):
+        self.level = level
+
+    def acquire(self):
+        pass  # pragma: NO COVER
+
+    def release(self):
+        pass  # pragma: NO COVER
+
+
+class _Logger(object):
+
+    def log_struct(self, message, severity=None):
+        self.log_struct_called_with = (message, severity)
+
+
+class _Client(object):
+
+    def __init__(self, project):
+        self.project = project
+        self.logger_ = _Logger()
+
+    def logger(self, _):  # pylint: disable=unused-argument
+        return self.logger_
+
+
+class _Record(object):
+
+    def __init__(self, name, level, message):
+        self.name = name
+        self.levelname = level
+        self.message = message
+        self.exc_info = None
+        self.exc_text = None
+        self.stack_info = None
+
+    def getMessage(self):
+        return self.message

system_tests/logging_.py

@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import time
 
 import unittest2
 
 from gcloud import _helpers
 from gcloud.environment_vars import TESTS_PROJECT
-from gcloud import logging
+import gcloud.logging
+import gcloud.logging.handlers
 
 from system_test_utils import unique_resource_id
 
@@ -71,13 +73,14 @@ class Config(object):
 
 def setUpModule():
     _helpers.PROJECT = TESTS_PROJECT
-    Config.CLIENT = logging.Client()
+    Config.CLIENT = gcloud.logging.Client()
 
 
 class TestLogging(unittest2.TestCase):
 
     def setUp(self):
         self.to_delete = []
+        self._handlers_cache = logging.getLogger().handlers[:]
 
     def tearDown(self):
         from gcloud.exceptions import NotFound
@@ -92,6 +95,7 @@ def tearDown(self):
                         time.sleep(backoff_intervals.pop(0))
                     else:
                         raise
+        logging.getLogger().handlers = self._handlers_cache[:]
 
     @staticmethod
     def _logger_name():
@@ -103,6 +107,7 @@ def test_log_text(self):
         self.to_delete.append(logger)
         logger.log_text(TEXT_PAYLOAD)
         entries, _ = _retry_backoff(_has_entries, logger.list_entries)
+
         self.assertEqual(len(entries), 1)
         self.assertEqual(entries[0].payload, TEXT_PAYLOAD)
 
@@ -151,6 +156,35 @@ def test_log_struct(self):
         self.assertEqual(len(entries), 1)
         self.assertEqual(entries[0].payload, JSON_PAYLOAD)
 
+    def test_log_handler(self):
+        LOG_MESSAGE = 'It was the best of times.'
+        # only create the logger to delete, hidden otherwise
+        logger = Config.CLIENT.logger(self._logger_name())
+        self.to_delete.append(logger)
+
+        handler = gcloud.logging.handlers.CloudLoggingHandler(Config.CLIENT)
+        cloud_logger = logging.getLogger(self._logger_name())
+        cloud_logger.addHandler(handler)
+        cloud_logger.warn(LOG_MESSAGE)
+        time.sleep(5)
+        entries, _ = logger.list_entries()
+        self.assertEqual(len(entries), 1)
+        self.assertEqual(entries[0].payload, {'message': LOG_MESSAGE})
+
+    def test_log_root_handler(self):
+        LOG_MESSAGE = 'It was the best of times.'
+        # only create the logger to delete, hidden otherwise
+        logger = Config.CLIENT.logger("root")
+        self.to_delete.append(logger)
+
+        handler = gcloud.logging.handlers.CloudLoggingHandler(Config.CLIENT)
+        gcloud.logging.handlers.setup_logging(handler)
+        logging.warn(LOG_MESSAGE)
+        time.sleep(5)
+        entries, _ = logger.list_entries()
+        self.assertEqual(len(entries), 1)
+        self.assertEqual(entries[0].payload, {'message': LOG_MESSAGE})
+
     def test_log_struct_w_metadata(self):
         JSON_PAYLOAD = {
             'message': 'System test: test_log_struct',