diff --git a/.travis.yml b/.travis.yml index 505b9ff..0302aa1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.6" - "2.7" - "3.2" - "3.3" diff --git a/README.md b/README.md index a905ca8..ca3a2e4 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ This client-library also has FluentHandler class for Python logging module. logging.basicConfig(level=logging.INFO) l = logging.getLogger('fluent.test') - l.addHandler(handler.FluentHandler('app.follow', host='host', port=24224)) + h = handler.FluentHandler('app.follow', host='host', port=24224) + h.setFormatter(handler.FluentRecordFormatter()) + l.addHandler(h) l.info({ 'from': 'userA', 'to': 'userB' diff --git a/fluent/handler.py b/fluent/handler.py index e14c036..308843d 100644 --- a/fluent/handler.py +++ b/fluent/handler.py @@ -2,6 +2,7 @@ import logging import socket +import sys try: import simplejson as json @@ -16,35 +17,65 @@ from fluent import sender -class FluentRecordFormatter(object): - def __init__(self): +class FluentRecordFormatter(logging.Formatter, object): + """ A structured formatter for Fluent. + + Best used with server storing data in an ElasticSearch cluster for example. + + :param fmt: a dict with format string as values to map to provided keys. + """ + def __init__(self, fmt=None, datefmt=None): + super(FluentRecordFormatter, self).__init__(None, datefmt) + + if not fmt: + self._fmt_dict = { + 'sys_host': '%(hostname)s', + 'sys_name': '%(name)s', + 'sys_module': '%(module)s', + } + else: + self._fmt_dict = fmt + self.hostname = socket.gethostname() def format(self, record): - data = {'sys_host': self.hostname, - 'sys_name': record.name, - 'sys_module': record.module, - # 'sys_lineno': record.lineno, - # 'sys_levelno': record.levelno, - # 'sys_levelname': record.levelname, - # 'sys_filename': record.filename, - # 'sys_funcname': record.funcName, - # 'sys_exc_info': record.exc_info, - } - # if 'sys_exc_info' in data and data['sys_exc_info']: - # data['sys_exc_info'] = self.formatException(data['sys_exc_info']) + # Only needed for python2.6 + if sys.version_info[0:2] <= (2, 6) and self.usesTime(): + record.asctime = self.formatTime(record, self.datefmt) + + # Compute attributes handled by parent class. + super(FluentRecordFormatter, self).format(record) + # Add ours + record.hostname = self.hostname + # Apply format + data = dict([(key, value % record.__dict__) + for key, value in self._fmt_dict.items()]) self._structuring(data, record.msg) return data + def usesTime(self): + return any([value.find('%(asctime)') >= 0 + for value in self._fmt_dict.values()]) + def _structuring(self, data, msg): + """ Melds `msg` into `data`. + + :param data: dictionary to be sent to fluent server + :param msg: :class:`LogRecord`'s message to add to `data`. + `msg` can be a simple string for backward compatibility with + :mod:`logging` framework, a JSON encoded string or a dictionary + that will be merged into dictionary generated in :meth:`format. + """ if isinstance(msg, dict): self._add_dic(data, msg) - elif isinstance(msg, str): + elif isinstance(msg, basestring): try: self._add_dic(data, json.loads(str(msg))) except ValueError: - pass + self._add_dic(data, {'message': msg}) + else: + self._add_dic(data, {'message': msg}) @staticmethod def _add_dic(data, dic): diff --git a/setup.py b/setup.py index 8e739c2..cf3abd0 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name='fluent-logger', - version='0.3.5', + version='0.4.0.dev', description=desc, long_description=open(README).read(), package_dir={'fluent': 'fluent'}, diff --git a/tests/test_handler.py b/tests/test_handler.py index 7f944a8..2ad6460 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -44,3 +44,80 @@ def test_simple(self): eq('userB', data[0][2]['to']) self.assertTrue(data[0][1]) self.assertTrue(isinstance(data[0][1], int)) + + def test_custom_fmt(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter( + fluent.handler.FluentRecordFormatter(fmt={ + 'name': '%(name)s', + 'lineno': '%(lineno)d', + 'emitted_at': '%(asctime)s', + }) + ) + log.addHandler(handler) + log.info({'sample': 'value'}) + handler.close() + + data = self.get_data() + self.assertTrue('name' in data[0][2]) + self.assertEqual('fluent.test', data[0][2]['name']) + self.assertTrue('lineno' in data[0][2]) + self.assertTrue('emitted_at' in data[0][2]) + + def test_json_encoded_message(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter(fluent.handler.FluentRecordFormatter()) + log.addHandler(handler) + log.info('{"key": "hello world!", "param": "value"}') + handler.close() + + data = self.get_data() + self.assertTrue('key' in data[0][2]) + self.assertEqual('hello world!', data[0][2]['key']) + + def test_unstructured_message(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter(fluent.handler.FluentRecordFormatter()) + log.addHandler(handler) + log.info('hello world') + handler.close() + + data = self.get_data() + self.assertTrue('message' in data[0][2]) + self.assertEqual('hello world', data[0][2]['message']) + + def test_non_string_simple_message(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter(fluent.handler.FluentRecordFormatter()) + log.addHandler(handler) + log.info(42) + handler.close() + + data = self.get_data() + self.assertTrue('message' in data[0][2]) + + def test_non_string_dict_message(self): + handler = fluent.handler.FluentHandler('app.follow', port=self._port) + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('fluent.test') + handler.setFormatter(fluent.handler.FluentRecordFormatter()) + log.addHandler(handler) + log.info({42: 'root'}) + handler.close() + + data = self.get_data() + # For some reason, non-string keys are ignored + self.assertFalse(42 in data[0][2])