From 3c8fc0d83d5b0ae034ba709118b4a3ba2a746ec7 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Tue, 3 Feb 2015 15:52:54 +0100 Subject: [PATCH 1/8] Add developement marker in version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e739c2..cf3abd0 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name='fluent-logger', - version='0.3.5', + version='0.4.0.dev', description=desc, long_description=open(README).read(), package_dir={'fluent': 'fluent'}, From d45e76502225685f9697c6d0ac673e2a01f39723 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Aug 2014 13:08:42 +0200 Subject: [PATCH 2/8] Add support for customizing structured message Should solve concerns from issue #7. --- fluent/handler.py | 40 ++++++++++++++++++++++++++-------------- tests/test_handler.py | 20 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/fluent/handler.py b/fluent/handler.py index e14c036..dc6292f 100644 --- a/fluent/handler.py +++ b/fluent/handler.py @@ -16,23 +16,35 @@ from fluent import sender -class FluentRecordFormatter(object): - def __init__(self): +class FluentRecordFormatter(logging.Formatter, object): + """ A structured formatter for Fluent. + + Best used with server storing data in an ElasticSearch cluster for example. + + :param fmt: a dict with format string as values to map to provided keys. + """ + def __init__(self, fmt=None, datefmt=None): + super(FluentRecordFormatter, self).__init__(None, datefmt) + + if not fmt: + self._fmt_dict = { + 'sys_host': '%(hostname)s', + 'sys_name': '%(name)s', + 'sys_module': '%(module)s', + } + else: + self._fmt_dict = fmt + self.hostname = socket.gethostname() def format(self, record): - data = {'sys_host': self.hostname, - 'sys_name': record.name, - 'sys_module': record.module, - # 'sys_lineno': record.lineno, - # 'sys_levelno': record.levelno, - # 'sys_levelname': record.levelname, - # 'sys_filename': record.filename, - # 'sys_funcname': record.funcName, - # 'sys_exc_info': record.exc_info, - } - # if 'sys_exc_info' in data and data['sys_exc_info']: - # data['sys_exc_info'] = self.formatException(data['sys_exc_info']) + # Compute attributes handled by parent class. + super(FluentRecordFormatter, self).format(record) + # Add ours + record.hostname = self.hostname + # Apply format + data = dict([(key, value % record.__dict__) + for key, value in self._fmt_dict.items()]) self._structuring(data, record.msg) return data diff --git a/tests/test_handler.py b/tests/test_handler.py index 7f944a8..8fe0a0e 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -44,3 +44,23 @@ def test_simple(self): eq('userB', data[0][2]['to']) self.assertTrue(data[0][1]) self.assertTrue(isinstance(data[0][1], int)) + + def test_custom_fmt(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter( + fluent.handler.FluentRecordFormatter(fmt={ + 'name': '%(name)s', + 'lineno': '%(lineno)d', + }) + ) + log.addHandler(handler) + log.info({'sample': 'value'}) + handler.close() + + data = self.get_data() + self.assertTrue('name' in data[0][2]) + self.assertEqual('fluent.test', data[0][2]['name']) + self.assertTrue('lineno' in data[0][2]) From 130be8304973bac3e80952bf72ffaf5b5347fc03 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Aug 2014 12:50:55 +0200 Subject: [PATCH 3/8] Use basestring instead of str, like in FluentHandler._add_dic For consistency's sake. --- fluent/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent/handler.py b/fluent/handler.py index dc6292f..5a856bc 100644 --- a/fluent/handler.py +++ b/fluent/handler.py @@ -52,7 +52,7 @@ def format(self, record): def _structuring(self, data, msg): if isinstance(msg, dict): self._add_dic(data, msg) - elif isinstance(msg, str): + elif isinstance(msg, basestring): try: self._add_dic(data, json.loads(str(msg))) except ValueError: From 1cde07aa1b0db60fecb7c831128fee20e910658d Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Aug 2014 12:51:37 +0200 Subject: [PATCH 4/8] Add compatibility with regular (non structured) logging usage Part of #8. --- fluent/handler.py | 4 +++- tests/test_handler.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/fluent/handler.py b/fluent/handler.py index 5a856bc..c13b4d9 100644 --- a/fluent/handler.py +++ b/fluent/handler.py @@ -56,7 +56,9 @@ def _structuring(self, data, msg): try: self._add_dic(data, json.loads(str(msg))) except ValueError: - pass + self._add_dic(data, {'message': msg}) + else: + self._add_dic(data, {'message': msg}) @staticmethod def _add_dic(data, dic): diff --git a/tests/test_handler.py b/tests/test_handler.py index 8fe0a0e..dbeeeee 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -64,3 +64,44 @@ def test_custom_fmt(self): self.assertTrue('name' in data[0][2]) self.assertEqual('fluent.test', data[0][2]['name']) self.assertTrue('lineno' in data[0][2]) + + def test_unstructured_message(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter(fluent.handler.FluentRecordFormatter()) + log.addHandler(handler) + log.info('hello world') + handler.close() + + data = self.get_data() + self.assertTrue('message' in data[0][2]) + self.assertEqual('hello world', data[0][2]['message']) + + def test_non_string_simple_message(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter(fluent.handler.FluentRecordFormatter()) + log.addHandler(handler) + log.info(42) + handler.close() + + data = self.get_data() + self.assertTrue('message' in data[0][2]) + + def test_non_string_dict_message(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter(fluent.handler.FluentRecordFormatter()) + log.addHandler(handler) + log.info({42: 'root'}) + handler.close() + + data = self.get_data() + # For some reason, non-string keys are ignored + self.assertFalse(42 in data[0][2]) From d08a55717247a60a3e4bd34d60788f42d017434f Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Aug 2014 13:00:54 +0200 Subject: [PATCH 5/8] Add docstring to FluentHandler._structuring --- fluent/handler.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fluent/handler.py b/fluent/handler.py index c13b4d9..253bdd1 100644 --- a/fluent/handler.py +++ b/fluent/handler.py @@ -50,6 +50,14 @@ def format(self, record): return data def _structuring(self, data, msg): + """ Melds `msg` into `data`. + + :param data: dictionary to be sent to fluent server + :param msg: :class:`LogRecord`'s message to add to `data`. + `msg` can be a simple string for backward compatibility with + :mod:`logging` framework, a JSON encoded string or a dictionary + that will be merged into dictionary generated in :meth:`format. + """ if isinstance(msg, dict): self._add_dic(data, msg) elif isinstance(msg, basestring): From 6cfd06a651046e179bc70551b576176bce413bb3 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Aug 2014 15:49:41 +0200 Subject: [PATCH 6/8] Fix support for asctime attribute This does not work in python2.6 as it did not use usesTime to detect the presence of asctime in fmt. --- .travis.yml | 1 - fluent/handler.py | 9 +++++++++ tests/test_handler.py | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 505b9ff..0302aa1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.6" - "2.7" - "3.2" - "3.3" diff --git a/fluent/handler.py b/fluent/handler.py index 253bdd1..308843d 100644 --- a/fluent/handler.py +++ b/fluent/handler.py @@ -2,6 +2,7 @@ import logging import socket +import sys try: import simplejson as json @@ -38,6 +39,10 @@ def __init__(self, fmt=None, datefmt=None): self.hostname = socket.gethostname() def format(self, record): + # Only needed for python2.6 + if sys.version_info[0:2] <= (2, 6) and self.usesTime(): + record.asctime = self.formatTime(record, self.datefmt) + # Compute attributes handled by parent class. super(FluentRecordFormatter, self).format(record) # Add ours @@ -49,6 +54,10 @@ def format(self, record): self._structuring(data, record.msg) return data + def usesTime(self): + return any([value.find('%(asctime)') >= 0 + for value in self._fmt_dict.values()]) + def _structuring(self, data, msg): """ Melds `msg` into `data`. diff --git a/tests/test_handler.py b/tests/test_handler.py index dbeeeee..adfaa96 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -54,6 +54,7 @@ def test_custom_fmt(self): fluent.handler.FluentRecordFormatter(fmt={ 'name': '%(name)s', 'lineno': '%(lineno)d', + 'emitted_at': '%(asctime)s', }) ) log.addHandler(handler) @@ -64,6 +65,7 @@ def test_custom_fmt(self): self.assertTrue('name' in data[0][2]) self.assertEqual('fluent.test', data[0][2]['name']) self.assertTrue('lineno' in data[0][2]) + self.assertTrue('emitted_at' in data[0][2]) def test_unstructured_message(self): handler = fluent.handler.FluentHandler('app.follow', port=self._port) From 8cd00682c73929ba1356629d9c8f8e68065f6cec Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 25 Feb 2015 15:41:24 +0100 Subject: [PATCH 7/8] Add test for logging json encoded strings --- tests/test_handler.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_handler.py b/tests/test_handler.py index adfaa96..2ad6460 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -67,6 +67,20 @@ def test_custom_fmt(self): self.assertTrue('lineno' in data[0][2]) self.assertTrue('emitted_at' in data[0][2]) + def test_json_encoded_message(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter(fluent.handler.FluentRecordFormatter()) + log.addHandler(handler) + log.info('{"key": "hello world!", "param": "value"}') + handler.close() + + data = self.get_data() + self.assertTrue('key' in data[0][2]) + self.assertEqual('hello world!', data[0][2]['key']) + def test_unstructured_message(self): handler = fluent.handler.FluentHandler('app.follow', port=self._port) From 70d68cbcaf111fc424bb965de8a124833e05889e Mon Sep 17 00:00:00 2001 From: Mac Ryan Date: Fri, 12 Sep 2014 14:58:43 +0200 Subject: [PATCH 8/8] Update README to proper use of FluentRecordFormatter --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a905ca8..ca3a2e4 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ This client-library also has FluentHandler class for Python logging module. logging.basicConfig(level=logging.INFO) l = logging.getLogger('fluent.test') - l.addHandler(handler.FluentHandler('app.follow', host='host', port=24224)) + h = handler.FluentHandler('app.follow', host='host', port=24224) + h.setFormatter(handler.FluentRecordFormatter()) + l.addHandler(h) l.info({ 'from': 'userA', 'to': 'userB'