Merge branch 'master' into nanosecond-time

luhn · luhn · commit 3bf3d3a19c17 · 2017-03-20T18:51:25.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,4 @@
 /.tox
 /build
 /dist
+.idea/
diff --git a/.travis.yml b/.travis.yml
@@ -6,6 +6,10 @@ python:
   - "3.3"
   - "3.4"
   - "3.5"
+  - "3.6"
+  - "3.6-dev"
+  - "3.7-dev"
+  - "nightly"
 # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
 install:
     - "pip install -e ."
diff --git a/README.rst b/README.rst
@@ -110,7 +110,7 @@ Event-Based Interface
 This API is a wrapper for `sender.FluentSender`.
 
 First, you need to call ``sender.setup()`` to create global `sender.FluentSender` logger
-instance. This call needs to be called only once, at the beggining of
+instance. This call needs to be called only once, at the beginning of
 the application for example.
 
 Initialization code of Event-Based API is below:
@@ -188,7 +188,7 @@ module.
 
     logging.basicConfig(level=logging.INFO)
     l = logging.getLogger('fluent.test')
-    h = handler.FluentHandler('app.follow', host='host', port=24224)
+    h = handler.FluentHandler('app.follow', host='host', port=24224, buffer_overflow_handler=handler)
     formatter = handler.FluentRecordFormatter(custom_format)
     h.setFormatter(formatter)
     l.addHandler(h)
@@ -211,6 +211,18 @@ You can also customize formatter via logging.config.dictConfig
 
     logging.config.dictConfig(conf['logging'])
 
+You can inject your own custom proc to handle buffer overflow in the event of connection failure. This will mitigate the loss of data instead of simply throwing data away.
+
+.. code:: python
+
+    import msgpack
+    from io import BytesIO
+
+    def handler(pendings):
+        unpacker = msgpack.Unpacker(BytesIO(pendings))
+        for unpacked in unpacker:
+            print(unpacked)
+
 A sample configuration ``logging.yaml`` would be:
 
 .. code:: python
@@ -242,17 +254,18 @@ A sample configuration ``logging.yaml`` would be:
                 host: localhost
                 port: 24224
                 tag: test.logging
+                buffer_overflow_handler: handler
                 formatter: fluent_fmt
                 level: DEBUG
-            null:
+            none:
                 class: logging.NullHandler
 
         loggers:
             amqp:
-                handlers: [null]
+                handlers: [none]
                 propagate: False
             conf:
-                handlers: [null]
+                handlers: [none]
                 propagate: False
             '': # root logger
                 handlers: [console, fluent]
@@ -265,6 +278,21 @@ Testing
 Testing can be done using
 `nose <https://nose.readthedocs.org/en/latest/>`__.
 
+Release
+-------
+
+Need wheel package.
+
+.. code:: sh
+
+    $ pip install wheel
+
+After that, type following command:
+
+.. code:: sh
+
+    $ python setup.py clean sdist bdist_wheel upload
+
 Contributors
 ------------
 
diff --git a/fluent/handler.py b/fluent/handler.py
@@ -23,8 +23,12 @@ class FluentRecordFormatter(logging.Formatter, object):
     Best used with server storing data in an ElasticSearch cluster for example.
 
     :param fmt: a dict with format string as values to map to provided keys.
+    :param datefmt: strftime()-compatible date/time format string.
+    :param style: (NOT USED)
+    :param fill_missing_fmt_key: if True, do not raise a KeyError if the format
+        key is not found. Put None if not found.s
     """
-    def __init__(self, fmt=None, datefmt=None):
+    def __init__(self, fmt=None, datefmt=None, style='%', fill_missing_fmt_key=False):
         super(FluentRecordFormatter, self).__init__(None, datefmt)
 
         if not fmt:
@@ -38,6 +42,8 @@ def __init__(self, fmt=None, datefmt=None):
 
         self.hostname = socket.gethostname()
 
+        self.fill_missing_fmt_key = fill_missing_fmt_key
+
     def format(self, record):
         # Only needed for python2.6
         if sys.version_info[0:2] <= (2, 6) and self.usesTime():
@@ -47,9 +53,18 @@ def format(self, record):
         super(FluentRecordFormatter, self).format(record)
         # Add ours
         record.hostname = self.hostname
+
         # Apply format
-        data = dict([(key, value % record.__dict__)
-                     for key, value in self._fmt_dict.items()])
+        data = {}
+        for key, value in self._fmt_dict.items():
+            try:
+                value = value % record.__dict__
+            except KeyError as exc:
+                value = None
+                if not self.fill_missing_fmt_key:
+                    raise exc
+
+            data[key] = value
 
         self._structuring(data, record)
         return data
@@ -73,7 +88,11 @@ def _structuring(self, data, record):
             self._add_dic(data, msg)
         elif isinstance(msg, basestring):
             try:
-                self._add_dic(data, json.loads(str(msg)))
+                json_msg = json.loads(str(msg))
+                if isinstance(json_msg, dict):
+                    self._add_dic(data, json_msg)
+                else:
+                    self._add_dic(data, {'message': str(json_msg)})
             except ValueError:
                 msg = record.getMessage()
                 self._add_dic(data, {'message': msg})
@@ -96,12 +115,14 @@ def __init__(self,
                  host='localhost',
                  port=24224,
                  timeout=3.0,
-                 verbose=False):
+                 verbose=False,
+                 buffer_overflow_handler=None):
 
         self.tag = tag
         self.sender = sender.FluentSender(tag,
                                           host=host, port=port,
-                                          timeout=timeout, verbose=verbose)
+                                          timeout=timeout, verbose=verbose,
+                                          buffer_overflow_handler=buffer_overflow_handler)
         logging.Handler.__init__(self)
 
     def emit(self, record):
diff --git a/setup.cfg b/setup.cfg
@@ -6,3 +6,5 @@ cover-erase = 1
 cover-branches = 1
 cover-inclusive = 1
 cover-min-percentage = 70
+[bdist_wheel]
+universal = 1
diff --git a/setup.py b/setup.py
@@ -12,7 +12,7 @@
 
 setup(
   name='fluent-logger',
-  version='0.4.2',
+  version='0.4.6',
   description=desc,
   long_description=open(README).read(),
   package_dir={'fluent': 'fluent'},
diff --git a/tests/test_handler.py b/tests/test_handler.py
@@ -62,6 +62,48 @@ def test_custom_fmt(self):
         self.assertTrue('lineno' in data[0][2])
         self.assertTrue('emitted_at' in data[0][2])
 
+    def test_custom_field_raise_exception(self):
+        handler = fluent.handler.FluentHandler('app.follow', port=self._port)
+
+        logging.basicConfig(level=logging.INFO)
+        log = logging.getLogger('fluent.test')
+        handler.setFormatter(
+            fluent.handler.FluentRecordFormatter(fmt={
+                'name': '%(name)s',
+                'custom_field': '%(custom_field)s'
+            })
+        )
+        log.addHandler(handler)
+        with self.assertRaises(KeyError):
+            log.info({'sample': 'value'})
+        log.removeHandler(handler)
+        handler.close()
+
+    def test_custom_field_fill_missing_fmt_key_is_true(self):
+        handler = fluent.handler.FluentHandler('app.follow', port=self._port)
+
+        logging.basicConfig(level=logging.INFO)
+        log = logging.getLogger('fluent.test')
+        handler.setFormatter(
+            fluent.handler.FluentRecordFormatter(fmt={
+                'name': '%(name)s',
+                'custom_field': '%(custom_field)s'
+                },
+                fill_missing_fmt_key=True
+            )
+        )
+        log.addHandler(handler)
+        log.info({'sample': 'value'})
+        log.removeHandler(handler)
+        handler.close()
+
+        data = self.get_data()
+        self.assertTrue('name' in data[0][2])
+        self.assertEqual('fluent.test', data[0][2]['name'])
+        self.assertTrue('custom_field' in data[0][2])
+        # field defaults to none if not in log record
+        self.assertIsNone(data[0][2]['custom_field'])
+
     def test_json_encoded_message(self):
         handler = fluent.handler.FluentHandler('app.follow', port=self._port)
 
@@ -104,6 +146,19 @@ def test_unstructured_formatted_message(self):
         self.assertTrue('message' in data[0][2])
         self.assertEqual('hello world, you!', data[0][2]['message'])
 
+    def test_number_string_simple_message(self):
+        handler = fluent.handler.FluentHandler('app.follow', port=self._port)
+
+        logging.basicConfig(level=logging.INFO)
+        log = logging.getLogger('fluent.test')
+        handler.setFormatter(fluent.handler.FluentRecordFormatter())
+        log.addHandler(handler)
+        log.info("1")
+        handler.close()
+
+        data = self.get_data()
+        self.assertTrue('message' in data[0][2])
+
     def test_non_string_simple_message(self):
         handler = fluent.handler.FluentHandler('app.follow', port=self._port)
 

-Original file line number
+Diff line change
 /.tox
 /build
 /dist
 +.idea/