Skip to content

Commit 5b83235

Browse files
mcipperlymcipperly
committed
Merge branch 'master' of https://github.com/fluent/fluent-logger-python into ipv6test
2 parents 7889118 + ace80f4 commit 5b83235

File tree

10 files changed

+113
-41
lines changed

10 files changed

+113
-41
lines changed

‎.travis.yml

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,17 @@
11
sudo: false
22
language: python
33
python:
4-
- "2.7"
5-
- "3.4"
64
- "3.5"
75
- "3.6"
8-
- 3.6-dev
9-
- 3.7-dev
10-
- pypy
6+
- "3.7"
7+
- "3.8"
8+
- "3.9"
119
- pypy3
1210
- nightly
1311
# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
1412
install:
1513
- "pip install -e ."
16-
- "pip install 'coverage>=3.7,<3.8' coveralls"
14+
- "pip install 'coverage~=4.5.4' coveralls"
1715
script:
1816
- "PYTHONFAULTHANDLER=x timeout -sABRT 30s nosetests -vsd"
1917
after_success:
@@ -27,7 +25,7 @@ deploy:
2725
secure: CpNaj4F3TZvpP1aSJWidh/XexrWODV2sBdObrYU79Gyh9hFl6WLsA3JM9BfVsy9cGb/P/jP6ly4Z0/6qdIzZ5D6FPOB1B7rn5GZ2LAMOypRCA6W2uJbRjUU373Wut0p0OmQcMPto6XJsMlpvOEq+1uAq+LLAnAGEmmYTeskZebs=
2826
on:
2927
tags: true
30-
condition: '"$TRAVIS_PYTHON_VERSION" = "3.6" || "$TRAVIS_PYTHON_VERSION" = "2.7"'
28+
condition: '"$TRAVIS_PYTHON_VERSION" = "3.9" || "$TRAVIS_PYTHON_VERSION" = "2.7"'
3129
distributions: "sdist bdist_wheel"
3230

3331
matrix:

‎MANIFEST.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
include README.rst
22
include setup.py
33
include COPYING
4+
include test/*py

‎README.rst

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,10 @@ Python application.
2424
Requirements
2525
------------
2626

27-
- Python 2.7 or 3.4+
28-
- ``msgpack-python``
27+
- Python 3.5+
28+
- ``msgpack``
2929
- **IMPORTANT**: Version 0.8.0 is the last version supporting Python 2.6, 3.2 and 3.3
30+
- **IMPORTANT**: Version 0.9.6 is the last version supporting Python 2.7 and 3.4
3031

3132
Installation
3233
------------

‎fluent/asyncsender.py

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,7 @@
11
# -*- coding: utf-8 -*-
22

3-
from __future__ import print_function
4-
53
import threading
6-
7-
try:
8-
from queue import Queue, Full, Empty
9-
except ImportError:
10-
from Queue import Queue, Full, Empty
4+
from queue import Queue, Full, Empty
115

126
from fluent import sender
137
from fluent.sender import EventTime
@@ -55,6 +49,7 @@ def __init__(self,
5549
msgpack_kwargs=None,
5650
queue_maxsize=DEFAULT_QUEUE_MAXSIZE,
5751
queue_circular=DEFAULT_QUEUE_CIRCULAR,
52+
queue_overflow_handler=None,
5853
**kwargs):
5954
"""
6055
:param kwargs: This kwargs argument is not used in __init__. This will be removed in the next major version.
@@ -66,6 +61,10 @@ def __init__(self,
6661
**kwargs)
6762
self._queue_maxsize = queue_maxsize
6863
self._queue_circular = queue_circular
64+
if queue_circular and queue_overflow_handler:
65+
self._queue_overflow_handler = queue_overflow_handler
66+
else:
67+
self._queue_overflow_handler = self._queue_overflow_handler_default
6968

7069
self._thread_guard = threading.Event() # This ensures visibility across all variables
7170
self._closed = False
@@ -109,13 +108,15 @@ def _send(self, bytes_):
109108
if self._queue_circular and self._queue.full():
110109
# discard oldest
111110
try:
112-
self._queue.get(block=False)
111+
discarded_bytes = self._queue.get(block=False)
113112
except Empty: # pragma: no cover
114113
pass
114+
else:
115+
self._queue_overflow_handler(discarded_bytes)
115116
try:
116117
self._queue.put(bytes_, block=(not self._queue_circular))
117-
except Full: # pragma: no cover
118-
return False # this actually can't happen
118+
except Full: # pragma: no cover
119+
return False # this actually can't happen
119120

120121
return True
121122

@@ -132,5 +133,8 @@ def _send_loop(self):
132133
finally:
133134
self._close()
134135

136+
def _queue_overflow_handler_default(self, discarded_bytes):
137+
pass
138+
135139
def __exit__(self, exc_type, exc_val, exc_tb):
136140
self.close()

‎fluent/handler.py

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,6 @@
99
except ImportError: # pragma: no cover
1010
import json
1111

12-
try:
13-
basestring
14-
except NameError: # pragma: no cover
15-
basestring = (str, bytes)
16-
1712
from fluent import sender
1813

1914

@@ -120,7 +115,7 @@ def _structuring(self, data, record):
120115

121116
if isinstance(msg, dict):
122117
self._add_dic(data, msg)
123-
elif isinstance(msg, basestring):
118+
elif isinstance(msg, str):
124119
self._add_dic(data, self._format_msg(record, msg))
125120
else:
126121
self._add_dic(data, {'message': msg})
@@ -171,8 +166,8 @@ def _format_by_dict_uses_time(self):
171166
@staticmethod
172167
def _add_dic(data, dic):
173168
for key, value in dic.items():
174-
if isinstance(key, basestring):
175-
data[str(key)] = value
169+
if isinstance(key, str):
170+
data[key] = value
176171

177172

178173
class FluentHandler(logging.Handler):

‎fluent/sender.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
# -*- coding: utf-8 -*-
22

3-
from __future__ import print_function
4-
53
import errno
64
import socket
75
import struct

‎setup.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,31 +12,32 @@
1212

1313
setup(
1414
name='fluent-logger',
15-
version='0.9.3',
15+
version='0.10.0',
1616
description=desc,
1717
long_description=open(README).read(),
1818
package_dir={'fluent': 'fluent'},
1919
packages=['fluent'],
20-
install_requires=['msgpack'],
20+
install_requires=['msgpack>1.0'],
2121
author='Kazuki Ohta',
2222
author_email='kazuki.ohta@gmail.com',
23+
maintainer='Arcadiy Ivanov',
24+
maintainer_email='arcadiy@ivanov.biz',
2325
url='https://github.com/fluent/fluent-logger-python',
24-
download_url='http://pypi.python.org/pypi/fluent-logger/',
26+
download_url='https://pypi.org/project/fluent-logger/',
2527
license='Apache License, Version 2.0',
2628
classifiers=[
27-
'Programming Language :: Python :: 2',
28-
'Programming Language :: Python :: 2.7',
2929
'Programming Language :: Python :: 3',
30-
'Programming Language :: Python :: 3.4',
3130
'Programming Language :: Python :: 3.5',
3231
'Programming Language :: Python :: 3.6',
3332
'Programming Language :: Python :: 3.7',
33+
'Programming Language :: Python :: 3.8',
34+
'Programming Language :: Python :: 3.9',
3435
'Programming Language :: Python :: Implementation :: CPython',
3536
'Programming Language :: Python :: Implementation :: PyPy',
3637
'Development Status :: 5 - Production/Stable',
3738
'Topic :: System :: Logging',
3839
'Intended Audience :: Developers',
3940
],
40-
python_requires=">=2.7,!=3.0,!=3.1,!=3.2,!=3.3,<3.8",
41+
python_requires='>=3.5',
4142
test_suite='tests'
4243
)

‎tests/mockserver.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,7 @@ def run(self):
6666
def get_received(self):
6767
self.join()
6868
self._buf.seek(0)
69-
# TODO: have to process string encoding properly. currently we assume
70-
# that all encoding is utf-8.
71-
return list(Unpacker(self._buf, encoding='utf-8'))
69+
return list(Unpacker(self._buf))
7270

7371
def close(self):
7472

‎tests/test_asynchandler.py

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,17 @@
44
import sys
55
import unittest
66

7+
try:
8+
from unittest import mock
9+
except ImportError:
10+
import mock
11+
try:
12+
from unittest.mock import patch
13+
except ImportError:
14+
from mock import patch
15+
16+
17+
718
import fluent.asynchandler
819
import fluent.handler
920
from tests import mockserver
@@ -309,3 +320,68 @@ def test_simple(self):
309320
eq('userB', el[2]['to'])
310321
self.assertTrue(el[1])
311322
self.assertTrue(isinstance(el[1], int))
323+
324+
325+
class QueueOverflowException(BaseException):
326+
pass
327+
328+
329+
def queue_overflow_handler(discarded_bytes):
330+
raise QueueOverflowException(discarded_bytes)
331+
332+
333+
class TestHandlerWithCircularQueueHandler(unittest.TestCase):
334+
Q_SIZE = 1
335+
336+
def setUp(self):
337+
super(TestHandlerWithCircularQueueHandler, self).setUp()
338+
self._server = mockserver.MockRecvServer('localhost')
339+
self._port = self._server.port
340+
341+
def tearDown(self):
342+
self._server.close()
343+
344+
def get_handler_class(self):
345+
# return fluent.handler.FluentHandler
346+
return fluent.asynchandler.FluentHandler
347+
348+
def test_simple(self):
349+
handler = self.get_handler_class()('app.follow', port=self._port,
350+
queue_maxsize=self.Q_SIZE,
351+
queue_circular=True,
352+
queue_overflow_handler=queue_overflow_handler)
353+
with handler:
354+
def custom_full_queue():
355+
handler.sender._queue.put(b'Mock', block=True)
356+
return True
357+
358+
with patch.object(fluent.asynchandler.asyncsender.Queue, 'full', mock.Mock(side_effect=custom_full_queue)):
359+
self.assertEqual(handler.sender.queue_circular, True)
360+
self.assertEqual(handler.sender.queue_maxsize, self.Q_SIZE)
361+
362+
logging.basicConfig(level=logging.INFO)
363+
log = logging.getLogger('fluent.test')
364+
handler.setFormatter(fluent.handler.FluentRecordFormatter())
365+
log.addHandler(handler)
366+
367+
exc_counter = 0
368+
369+
try:
370+
log.info({'cnt': 1, 'from': 'userA', 'to': 'userB'})
371+
except QueueOverflowException:
372+
exc_counter += 1
373+
374+
try:
375+
log.info({'cnt': 2, 'from': 'userA', 'to': 'userB'})
376+
except QueueOverflowException:
377+
exc_counter += 1
378+
379+
try:
380+
log.info({'cnt': 3, 'from': 'userA', 'to': 'userB'})
381+
except QueueOverflowException:
382+
exc_counter += 1
383+
384+
# we can't be sure to have exception in every case due to multithreading,
385+
# so we can test only for a cautelative condition here
386+
print('Exception raised: {} (expected 3)'.format(exc_counter))
387+
assert exc_counter >= 0

‎tox.ini

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
[tox]
22
minversion = 1.7.2
3-
envlist = py27, py32, py33, py34, py35
3+
envlist = py27, py32, py33, py34, py35, py36, py37, py38
44
skip_missing_interpreters = True
55

66
[testenv]
77
deps = nose
8-
coverage>=3.7,<3.8
8+
coverage~=4.5.4
99
commands = python setup.py nosetests

0 commit comments

Comments
 (0)