Add ability to configure QueueHandler/QueueListener together and prov…

…ide a getHandlerByName API.
ambv · Jun 7, 2022 · May 25, 2022 · May 26, 2022 · May 27, 2022 · May 27, 2022
commit 3686d74fa2bc38b0d007645f457d0418ec098c96
@@ -661,6 +661,42 @@ it with :func:`staticmethod`. For example::
 You don't need to wrap with :func:`staticmethod` if you're setting the import
 callable on a configurator *instance*.
 
+.. _configure-queue:
+
+Configuring QueueHandler and QueueListener
+""""""""""""""""""""""""""""""""""""""""""
+
+If you want to configure a :class:`~logging.handlers.QueueHandler`, noting that this
+is normally used in conjunction with a :class:`~logging.handlers.QueueListener`, you
+can configure both together. After the configuration, the ``QueueListener`` instance
+will be available as the :attr:`listener` attribute of the created handler, and that
+in turn will be available to you using :func:`~logging.getHandlerByName` and passing
+whatever name you have used for the ``QueueHandler`` in your configuration. The
-whatever name you have used for the ``QueueHandler`` in your configuration. The
+the name you have used for the ``QueueHandler`` in your configuration. The
-whatever name you have used for the ``QueueHandler`` in your configuration. The
+the name you have used for the ``QueueHandler`` in your configuration. The
+dictionary schema for configuring the pair is shown in the example YAML snippet below.
+
+.. code-block:: yaml
+
+    handlers:
+      qhand:
+        class: logging.handlers.QueueHandler
+        queue: my.module.queuefactory
+        handlers:
+          - hand_name_1
+          - hand_name_2
+          ...
+
+If the ``queue`` key is provided, the value should resolve to a callable which returns
+a suitable :class:`queue.Queue` instance for use by the handler and listener. If it is
+not provided, a standard unbounded :class:`queue.Queue` instance is created and used.
+The values under the ``handlers`` key are the names of other handlers in the
+configuration (not shown in the above snippet) which will be passed to the
+``QueueListener``, which will be stored in the handler's :attr:`listener` attribute.
+You can customize the listener class with a ``listener`` key whose value is a string
+which resolves to a listener class, such as ``'mypackage.CustomListener'``. Any custom
+queue handler and listener classes will need to deal with the same initialization
+signatures as :class:`~logging.handlers.QueueHandler` and
+:class:`~logging.handlers.QueueListener`.
+
 
 .. _logging-config-fileformat:
 

@@ -1051,7 +1051,13 @@ possible, while any potentially slow operations (such as sending an email via
       want to override this if you want to use blocking behaviour, or a
       timeout, or a customized queue implementation.
 
+   .. attribute:: listener
 
+      When created via configuration using :func:`~logging.config.dictConfig`, this
+      attribute will contain a :class:`QueueListener` instance for use with this
+      handler. Otherwise, it will be ``None``.
+
+      .. versionadded:: 3.12
 
 .. _queue-listener:
 

@@ -1164,6 +1164,13 @@ functions.
       This undocumented behaviour was considered a mistake, and was removed in
       Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility.
 
+.. function:: getHandlerByName(name)
+
+   Returns a handler with the specified *name*, or ``None`` if there is no handler
+   with that name.
+
+   .. versionadded:: 3.12
+
 .. function:: makeLogRecord(attrdict)
 
    Creates and returns a new :class:`LogRecord` instance whose attributes are

@@ -38,7 +38,8 @@
            'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass',
            'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown',
            'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory',
-           'lastResort', 'raiseExceptions', 'getLevelNamesMapping']
+           'lastResort', 'raiseExceptions', 'getLevelNamesMapping',
+           'getHandlerByName']
 
 import threading
 
@@ -885,6 +886,15 @@ def _addHandlerRef(handler):
     finally:
         _releaseLock()
 
+
+def getHandlerByName(name):
+    """
+    Get a handler with the specified *name*, or None if there isn't one with
+    that name.
+    """
+    return _handlers.get(name)
+
+
 class Handler(Filterer):
     """
     Handler instances dispatch logging events to specific destinations.

@@ -25,9 +25,11 @@
 """
 
 import errno
+import functools
 import io
 import logging
 import logging.handlers
+import queue
 import re
 import struct
 import threading
@@ -563,7 +565,7 @@ def configure(self):
                         handler.name = name
                         handlers[name] = handler
                     except Exception as e:
-                        if 'target not configured yet' in str(e.__cause__):
+                        if ' not configured yet' in str(e.__cause__):
                             deferred.append(name)
                         else:
                             raise ValueError('Unable to configure handler '
@@ -702,6 +704,21 @@ def add_filters(self, filterer, filters):
             except Exception as e:
                 raise ValueError('Unable to add filter %r' % f) from e
 
+    def _configure_queue_handler(self, klass, **kwargs):
+        if 'queue' in kwargs:
+            q = kwargs['queue']
+        else:
+            q = queue.Queue()  # unbounded
+        rhl = kwargs.get('respect_handler_level', False)
+        if 'listener' in kwargs:
+            lklass = kwargs['listener']
+        else:
+            lklass = logging.handlers.QueueListener
+        listener = lklass(q, *kwargs['handlers'], respect_handler_level=rhl)
+        handler = klass(q)
+        handler.listener = listener
+        return handler
+
     def configure_handler(self, config):
         """Configure a handler from a dictionary."""
         config_copy = dict(config)  # for restoring in case of error
@@ -721,26 +738,61 @@ def configure_handler(self, config):
             factory = c
         else:
             cname = config.pop('class')
-            klass = self.resolve(cname)
-            #Special case for handler which refers to another handler
+            if callable(cname):
+                klass = cname
+            else:
+                klass = self.resolve(cname)
             if issubclass(klass, logging.handlers.MemoryHandler) and\
                 'target' in config:
+                # Special case for handler which refers to another handler
                 try:
-                    th = self.config['handlers'][config['target']]
+                    tn = config['target']
+                    th = self.config['handlers'][tn]
                     if not isinstance(th, logging.Handler):
                         config.update(config_copy)  # restore for deferred cfg
                         raise TypeError('target not configured yet')
                     config['target'] = th
                 except Exception as e:
-                    raise ValueError('Unable to set target handler '
-                                     '%r' % config['target']) from e
+                    raise ValueError('Unable to set target handler %r' % tn) from e
+            elif issubclass(klass, logging.handlers.QueueHandler):
+                # Another special case for handler which refers to other handlers
+                if 'handlers' not in config:
+                    raise ValueError('No handlers specified for a QueueHandler')
+                if 'queue' in config and\
+                    not isinstance(qspec := config['queue'], queue.Queue):
+                    q = self.resolve(qspec)
+                    if not callable(q):
+                        raise TypeError('Invalid queue specifier %r' % qspec)
+                    config['queue'] = q()
+                if 'listener' in config:
+                    lspec = config['listener']
+                    if isinstance(lspec, str):
+                        listener = self.resolve(lspec)
+                        config['listener'] = listener
+                    elif not issubclass(lspec, logging.handlers.QueueListener):
+                        raise TypeError('Invalid listener spec %r' % lspec)
+                hlist = []
+                try:
+                    for hn in config['handlers']:
+                        h = self.config['handlers'][hn]
+                        if not isinstance(h, logging.Handler):
+                            config.update(config_copy)  # restore for deferred cfg
+                            raise TypeError('needed handler %r '
-                            raise TypeError('needed handler %r '
+                            raise TypeError('Required handler %r '
-                            raise TypeError('needed handler %r '
+                            raise TypeError('Required handler %r '
+                                            'is not configured yet' % hn)
+                        hlist.append(h)
+                except Exception as e:
+                    raise ValueError('Unable to set required handler %r' % hn) from e
+                config['handlers'] = hlist
             elif issubclass(klass, logging.handlers.SMTPHandler) and\
                 'mailhost' in config:
                 config['mailhost'] = self.as_tuple(config['mailhost'])
             elif issubclass(klass, logging.handlers.SysLogHandler) and\
                 'address' in config:
                 config['address'] = self.as_tuple(config['address'])
-            factory = klass
+            if issubclass(klass, logging.handlers.QueueHandler):
+                factory = functools.partial(self._configure_queue_handler, klass)
+            else:
+                factory = klass
         props = config.pop('.', None)
         kwargs = {k: config[k] for k in config if valid_ident(k)}
         try:

@@ -1424,6 +1424,7 @@ def __init__(self, queue):
         """
         logging.Handler.__init__(self)
         self.queue = queue
+        self.listener = None  # will be set to listener if configured via dictConfig()
 
     def enqueue(self, record):
         """